From 1bf57eb7b07dd19bcad64d37cd4a44c7161b3962 Mon Sep 17 00:00:00 2001 From: Lucas Pinheiro Date: Mon, 7 Oct 2024 17:11:29 -0300 Subject: [PATCH 1/4] feat: expose creation timestamp filters for pipelines/workflows --- api/client/pipelines_v1_alpha.go | 31 ++++++ api/client/workflows_v1_alpha.go | 25 +++++ cmd/get.go | 37 ++++++- cmd/get_test.go | 174 ++++++++++++++++++++++++++++++- cmd/pipelines/get.go | 4 +- cmd/workflows/get.go | 4 +- 6 files changed, 267 insertions(+), 8 deletions(-) diff --git a/api/client/pipelines_v1_alpha.go b/api/client/pipelines_v1_alpha.go index 634a254..a8a8f0e 100644 --- a/api/client/pipelines_v1_alpha.go +++ b/api/client/pipelines_v1_alpha.go @@ -3,6 +3,7 @@ package client import ( "errors" "fmt" + "net/url" models "github.com/semaphoreci/cli/api/models" "github.com/semaphoreci/cli/api/uuid" @@ -92,6 +93,11 @@ func (c *PipelinesApiV1AlphaApi) ListPplByWfID(projectID, wfID string) ([]byte, return body, nil } +type ListOptions struct { + CreatedAfter int64 + CreatedBefore int64 +} + func (c *PipelinesApiV1AlphaApi) ListPpl(projectID string) ([]byte, error) { detailed := fmt.Sprintf("%s?project_id=%s", c.ResourceNamePlural, projectID) body, status, err := c.BaseClient.List(detailed) @@ -106,3 +112,28 @@ func (c *PipelinesApiV1AlphaApi) ListPpl(projectID string) ([]byte, error) { return body, nil } + +func (c *PipelinesApiV1AlphaApi) ListPplWithOptions(projectID string, options ListOptions) ([]byte, error) { + query := url.Values{} + query.Add("project_id", projectID) + + if options.CreatedAfter > 0 { + query.Add("created_after", fmt.Sprintf("%d", options.CreatedAfter)) + } + + if options.CreatedBefore > 0 { + query.Add("created_before", fmt.Sprintf("%d", options.CreatedBefore)) + } + + body, status, err := c.BaseClient.ListWithParams(c.ResourceNamePlural, query) + + if err != nil { + return nil, errors.New(fmt.Sprintf("connecting to Semaphore failed '%s'", err)) + } + + if status != 200 { + return nil, errors.New(fmt.Sprintf("http status %d with message \"%s\" received from upstream", status, body)) + } + + return body, nil +} diff --git a/api/client/workflows_v1_alpha.go b/api/client/workflows_v1_alpha.go index 723de2d..f2a738a 100644 --- a/api/client/workflows_v1_alpha.go +++ b/api/client/workflows_v1_alpha.go @@ -3,6 +3,7 @@ package client import ( "errors" "fmt" + "net/url" models "github.com/semaphoreci/cli/api/models" "github.com/semaphoreci/cli/api/uuid" @@ -40,6 +41,30 @@ func (c *WorkflowApiV1AlphaApi) ListWorkflows(project_id string) (*models.Workfl return models.NewWorkflowListV1AlphaFromJson(body) } +func (c *WorkflowApiV1AlphaApi) ListWorkflowsWithOptions(projectID string, options ListOptions) (*models.WorkflowListV1Alpha, error) { + query := url.Values{} + query.Add("project_id", projectID) + + if options.CreatedAfter > 0 { + query.Add("created_after", fmt.Sprintf("%d", options.CreatedAfter)) + } + + if options.CreatedBefore > 0 { + query.Add("created_before", fmt.Sprintf("%d", options.CreatedBefore)) + } + + body, status, err := c.BaseClient.ListWithParams(c.ResourceNamePlural, query) + if err != nil { + return nil, errors.New(fmt.Sprintf("connecting to Semaphore failed '%s'", err)) + } + + if status != 200 { + return nil, errors.New(fmt.Sprintf("http status %d with message \"%s\" received from upstream", status, body)) + } + + return models.NewWorkflowListV1AlphaFromJson(body) +} + func (c *WorkflowApiV1AlphaApi) CreateSnapshotWf(project_id, label, archivePath string) ([]byte, error) { requestToken, err := uuid.NewUUID() diff --git a/cmd/get.go b/cmd/get.go index 9abb0b1..1c09e43 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -5,6 +5,7 @@ import ( "log" "os" "text/tabwriter" + "time" client "github.com/semaphoreci/cli/api/client" models "github.com/semaphoreci/cli/api/models" @@ -398,7 +399,7 @@ var GetPplCmd = &cobra.Command{ if len(args) == 0 { projectID := getPrj(cmd) - pipelines.List(projectID) + pipelines.List(projectID, listOptions(cmd)) } else { id := args[0] pipelines.Describe(id, GetPplFollow) @@ -417,7 +418,7 @@ var GetWfCmd = &cobra.Command{ projectID := getPrj(cmd) if len(args) == 0 { - workflows.List(projectID) + workflows.List(projectID, listOptions(cmd)) } else { wfID := args[0] workflows.Describe(projectID, wfID) @@ -504,6 +505,30 @@ func getPrj(cmd *cobra.Command) string { return projectID } +func listOptions(cmd *cobra.Command) client.ListOptions { + // + // By default, we return resources (pipelines/workflows) created in the last 3 months. + // + options := client.ListOptions{ + CreatedAfter: time.Now().Add(-1 * time.Hour * 24 * 90).Unix(), + CreatedBefore: time.Now().Unix(), + } + + createdAfter, err := cmd.Flags().GetInt64("created-after") + utils.Check(err) + if createdAfter > 0 { + options.CreatedAfter = createdAfter + } + + createdBefore, err := cmd.Flags().GetInt64("created-before") + utils.Check(err) + if createdBefore > 0 { + options.CreatedBefore = createdBefore + } + + return options +} + func init() { RootCmd.AddCommand(getCmd) @@ -533,6 +558,10 @@ func init() { "project name; if not specified will be inferred from git origin") GetPplCmd.Flags().StringP("project-id", "i", "", "project id; if not specified will be inferred from git origin") + GetPplCmd.Flags().Int64P("created-after", "", 0, + "filter for pipeline creation timestamp; by default, this is (now - 90d)") + GetPplCmd.Flags().Int64P("created-before", "", 0, + "filter for pipeline creation timestamp; by default, this is now") getCmd.AddCommand(GetPplCmd) getCmd.AddCommand(GetWfCmd) @@ -540,6 +569,10 @@ func init() { "project name; if not specified will be inferred from git origin") GetWfCmd.Flags().StringP("project-id", "i", "", "project id; if not specified will be inferred from git origin") + GetWfCmd.Flags().Int64P("created-after", "", 0, + "filter for pipeline creation timestamp; by default, this is (now - 90d)") + GetWfCmd.Flags().Int64P("created-before", "", 0, + "filter for pipeline creation timestamp; by default, this is now") getCmd.AddCommand(GetDTCmd) GetDTCmd.Flags().StringP("project-name", "p", "", diff --git a/cmd/get_test.go b/cmd/get_test.go index 5f3cdf7..28f709a 100644 --- a/cmd/get_test.go +++ b/cmd/get_test.go @@ -5,6 +5,7 @@ import ( "net/http" "net/url" "testing" + "time" httpmock "github.com/jarcoal/httpmock" "github.com/stretchr/testify/assert" @@ -360,6 +361,114 @@ func Test__GetAgent__Response200(t *testing.T) { } } +func Test__GetPipelines__Response200(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + received := false + + httpmock.RegisterResponder("GET", "https://org.semaphoretext.xyz/api/v1alpha/projects/foo", + func(req *http.Request) (*http.Response, error) { + received = true + + p := `{ + "metadata": { + "id": "758cb945-7495-4e40-a9a1-4b3991c6a8fe" + } + }` + + return httpmock.NewStringResponse(200, p), nil + }, + ) + + httpmock.RegisterResponder("GET", `=~^https:\/\/org\.semaphoretext\.xyz\/api\/v1alpha\/pipelines\?created_after=\d+&created_before=\d+&project_id=758cb945-7495-4e40-a9a1-4b3991c6a8fe`, + func(req *http.Request) (*http.Response, error) { + received = true + + p := `[{ + "pipeline": { + "ppl_id": "494b76aa-f3f0-4ecf-b5ef-c389591a01be", + "name": "snapshot test", + "state": "done", + "result": "passed", + "result_reason": "test", + "error_description": "" + } + }]` + + return httpmock.NewStringResponse(200, p), nil + }, + ) + + RootCmd.SetArgs([]string{"get", "pipelines", "--project-name", "foo"}) + RootCmd.Execute() + + if received == false { + t.Error("Expected the API to receive GET /pipelines") + } +} + +func Test__GetPipelines__WithCreationTimestampFilters(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + received := false + + httpmock.RegisterResponder("GET", "https://org.semaphoretext.xyz/api/v1alpha/projects/foo", + func(req *http.Request) (*http.Response, error) { + received = true + + p := `{ + "metadata": { + "id": "758cb945-7495-4e40-a9a1-4b3991c6a8fe" + } + }` + + return httpmock.NewStringResponse(200, p), nil + }, + ) + + threeMonthsAgo := fmt.Sprintf("%d", time.Now().Add(-1*time.Hour*24*90).Unix()) + oneMonthAgo := fmt.Sprintf("%d", time.Now().Add(-1*time.Hour*24*30).Unix()) + + url := fmt.Sprintf("https://org.semaphoretext.xyz/api/v1alpha/pipelines?created_after=%s&created_before=%s&project_id=758cb945-7495-4e40-a9a1-4b3991c6a8fe", threeMonthsAgo, oneMonthAgo) + httpmock.RegisterResponder("GET", url, + func(req *http.Request) (*http.Response, error) { + received = true + + p := `[{ + "pipeline": { + "ppl_id": "494b76aa-f3f0-4ecf-b5ef-c389591a01be", + "name": "snapshot test", + "state": "done", + "result": "passed", + "result_reason": "test", + "error_description": "" + } + }]` + + return httpmock.NewStringResponse(200, p), nil + }, + ) + + RootCmd.SetArgs([]string{ + "get", + "pipelines", + "--project-name", + "foo", + "--created-after", + threeMonthsAgo, + "--created-before", + oneMonthAgo, + }) + + RootCmd.Execute() + + if received == false { + t.Error("Expected the API to receive GET /pipelines") + } +} + func Test__GetPipeline__Response200(t *testing.T) { httpmock.Activate() defer httpmock.DeactivateAndReset() @@ -537,7 +646,7 @@ func Test__GetWorkflows__Response200(t *testing.T) { }, ) - httpmock.RegisterResponder("GET", "https://org.semaphoretext.xyz/api/v1alpha/plumber-workflows?project_id=758cb945-7495-4e40-a9a1-4b3991c6a8fe", + httpmock.RegisterResponder("GET", `=~^https:\/\/org\.semaphoretext\.xyz\/api\/v1alpha\/plumber-workflows\?created_after=\d+&created_before=\d+&project_id=758cb945-7495-4e40-a9a1-4b3991c6a8fe`, func(req *http.Request) (*http.Response, error) { received = true @@ -560,6 +669,67 @@ func Test__GetWorkflows__Response200(t *testing.T) { RootCmd.Execute() if received == false { - t.Error("Expected the API to receive GET secrets/aaaaaaa") + t.Error("Expected the API to receive GET /plumber-workflows") + } +} + +func Test__GetWorkflows__WithCreationTimestampFilters(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + received := false + + httpmock.RegisterResponder("GET", "https://org.semaphoretext.xyz/api/v1alpha/projects/foo", + func(req *http.Request) (*http.Response, error) { + received = true + + p := `{ + "metadata": { + "id": "758cb945-7495-4e40-a9a1-4b3991c6a8fe" + } + }` + + return httpmock.NewStringResponse(200, p), nil + }, + ) + + threeMonthsAgo := fmt.Sprintf("%d", time.Now().Add(-1*time.Hour*24*90).Unix()) + oneMonthAgo := fmt.Sprintf("%d", time.Now().Add(-1*time.Hour*24*30).Unix()) + + url := fmt.Sprintf("https://org.semaphoretext.xyz/api/v1alpha/plumber-workflows?created_after=%s&created_before=%s&project_id=758cb945-7495-4e40-a9a1-4b3991c6a8fe", threeMonthsAgo, oneMonthAgo) + httpmock.RegisterResponder("GET", url, + func(req *http.Request) (*http.Response, error) { + received = true + + p := `[{ + "wf_id": "b129e277-4aa5-4308-8e31-ec825815e335", + "requester_id": "92f81b82-3584-4852-ab28-4866624bed1e", + "project_id": "758cb945-7495-4e40-a9a1-4b3991c6a8fe", + "initial_ppl_id": "92f81b82-3584-4852-ab28-4866624bed1e", + "created_at": { + "seconds": 1533833523, + "nanos": 537460000 + } + }]` + + return httpmock.NewStringResponse(200, p), nil + }, + ) + + RootCmd.SetArgs([]string{ + "get", + "workflows", + "--project-name", + "foo", + "--created-after", + threeMonthsAgo, + "--created-before", + oneMonthAgo, + }) + + RootCmd.Execute() + + if received == false { + t.Error("Expected the API to receive GET /plumber-workflows") } } diff --git a/cmd/pipelines/get.go b/cmd/pipelines/get.go index 24fc6c1..e610caf 100644 --- a/cmd/pipelines/get.go +++ b/cmd/pipelines/get.go @@ -37,10 +37,10 @@ func describe(c client.PipelinesApiV1AlphaApi, id string) ([]byte, bool) { return pplY, pplJ.IsDone() } -func List(projectID string) { +func List(projectID string, options client.ListOptions) { fmt.Printf("%s\n", projectID) c := client.NewPipelinesV1AlphaApi() - body, err := c.ListPpl(projectID) + body, err := c.ListPplWithOptions(projectID, options) utils.Check(err) prettyPrintPipelineList(body) diff --git a/cmd/workflows/get.go b/cmd/workflows/get.go index 8cf6c01..2a4ed0d 100644 --- a/cmd/workflows/get.go +++ b/cmd/workflows/get.go @@ -11,9 +11,9 @@ import ( "github.com/semaphoreci/cli/cmd/utils" ) -func List(projectID string) { +func List(projectID string, options client.ListOptions) { wfClient := client.NewWorkflowV1AlphaApi() - workflows, err := wfClient.ListWorkflows(projectID) + workflows, err := wfClient.ListWorkflowsWithOptions(projectID, options) utils.Check(err) prettyPrint(workflows) From 1c3a10a36a7e8986d2ebe5a939ac5f787fbea86f Mon Sep 17 00:00:00 2001 From: Lucas Pinheiro Date: Tue, 8 Oct 2024 08:19:02 -0300 Subject: [PATCH 2/4] retrigger build From ff9cfe8caedb09f16a874398d56a6656629e72d7 Mon Sep 17 00:00:00 2001 From: Lucas Pinheiro Date: Tue, 8 Oct 2024 12:18:45 -0300 Subject: [PATCH 3/4] use --age argument instead --- cmd/get.go | 38 ++++++++++++-------------------------- cmd/get_test.go | 28 ++++++++++++---------------- 2 files changed, 24 insertions(+), 42 deletions(-) diff --git a/cmd/get.go b/cmd/get.go index 1c09e43..c61e3c5 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -17,6 +17,10 @@ import ( "github.com/spf13/cobra" ) +const ( + DefaultListingAge = time.Hour * 24 * 90 +) + var getCmd = &cobra.Command{ Use: "get [KIND]", Short: "List resources.", @@ -506,27 +510,13 @@ func getPrj(cmd *cobra.Command) string { } func listOptions(cmd *cobra.Command) client.ListOptions { - // - // By default, we return resources (pipelines/workflows) created in the last 3 months. - // - options := client.ListOptions{ - CreatedAfter: time.Now().Add(-1 * time.Hour * 24 * 90).Unix(), - CreatedBefore: time.Now().Unix(), - } - - createdAfter, err := cmd.Flags().GetInt64("created-after") + age, err := cmd.Flags().GetDuration("age") utils.Check(err) - if createdAfter > 0 { - options.CreatedAfter = createdAfter - } - createdBefore, err := cmd.Flags().GetInt64("created-before") - utils.Check(err) - if createdBefore > 0 { - options.CreatedBefore = createdBefore + return client.ListOptions{ + CreatedBefore: time.Now().Unix(), + CreatedAfter: time.Now().Add(-1 * age).Unix(), } - - return options } func init() { @@ -558,10 +548,8 @@ func init() { "project name; if not specified will be inferred from git origin") GetPplCmd.Flags().StringP("project-id", "i", "", "project id; if not specified will be inferred from git origin") - GetPplCmd.Flags().Int64P("created-after", "", 0, - "filter for pipeline creation timestamp; by default, this is (now - 90d)") - GetPplCmd.Flags().Int64P("created-before", "", 0, - "filter for pipeline creation timestamp; by default, this is now") + GetPplCmd.Flags().DurationP("age", "", DefaultListingAge, + "filter for listing pipelines based on age; by default, we list only pipelines created in the last 90 days") getCmd.AddCommand(GetPplCmd) getCmd.AddCommand(GetWfCmd) @@ -569,10 +557,8 @@ func init() { "project name; if not specified will be inferred from git origin") GetWfCmd.Flags().StringP("project-id", "i", "", "project id; if not specified will be inferred from git origin") - GetWfCmd.Flags().Int64P("created-after", "", 0, - "filter for pipeline creation timestamp; by default, this is (now - 90d)") - GetWfCmd.Flags().Int64P("created-before", "", 0, - "filter for pipeline creation timestamp; by default, this is now") + GetWfCmd.Flags().DurationP("age", "", DefaultListingAge, + "filter for listing pipelines based on age; by default, we list only pipelines created in the last 90 days") getCmd.AddCommand(GetDTCmd) GetDTCmd.Flags().StringP("project-name", "p", "", diff --git a/cmd/get_test.go b/cmd/get_test.go index 28f709a..8796231 100644 --- a/cmd/get_test.go +++ b/cmd/get_test.go @@ -428,10 +428,10 @@ func Test__GetPipelines__WithCreationTimestampFilters(t *testing.T) { }, ) - threeMonthsAgo := fmt.Sprintf("%d", time.Now().Add(-1*time.Hour*24*90).Unix()) - oneMonthAgo := fmt.Sprintf("%d", time.Now().Add(-1*time.Hour*24*30).Unix()) - - url := fmt.Sprintf("https://org.semaphoretext.xyz/api/v1alpha/pipelines?created_after=%s&created_before=%s&project_id=758cb945-7495-4e40-a9a1-4b3991c6a8fe", threeMonthsAgo, oneMonthAgo) + age := 720 * time.Hour + createdBefore := fmt.Sprintf("%d", time.Now().Unix()) + createdAfter := fmt.Sprintf("%d", time.Now().Add(-1*age).Unix()) + url := fmt.Sprintf("https://org.semaphoretext.xyz/api/v1alpha/pipelines?created_after=%s&created_before=%s&project_id=758cb945-7495-4e40-a9a1-4b3991c6a8fe", createdAfter, createdBefore) httpmock.RegisterResponder("GET", url, func(req *http.Request) (*http.Response, error) { received = true @@ -456,10 +456,8 @@ func Test__GetPipelines__WithCreationTimestampFilters(t *testing.T) { "pipelines", "--project-name", "foo", - "--created-after", - threeMonthsAgo, - "--created-before", - oneMonthAgo, + "--age", + age.String(), }) RootCmd.Execute() @@ -693,10 +691,10 @@ func Test__GetWorkflows__WithCreationTimestampFilters(t *testing.T) { }, ) - threeMonthsAgo := fmt.Sprintf("%d", time.Now().Add(-1*time.Hour*24*90).Unix()) - oneMonthAgo := fmt.Sprintf("%d", time.Now().Add(-1*time.Hour*24*30).Unix()) - - url := fmt.Sprintf("https://org.semaphoretext.xyz/api/v1alpha/plumber-workflows?created_after=%s&created_before=%s&project_id=758cb945-7495-4e40-a9a1-4b3991c6a8fe", threeMonthsAgo, oneMonthAgo) + age := 720 * time.Hour + createdBefore := fmt.Sprintf("%d", time.Now().Unix()) + createdAfter := fmt.Sprintf("%d", time.Now().Add(-1*age).Unix()) + url := fmt.Sprintf("https://org.semaphoretext.xyz/api/v1alpha/plumber-workflows?created_after=%s&created_before=%s&project_id=758cb945-7495-4e40-a9a1-4b3991c6a8fe", createdAfter, createdBefore) httpmock.RegisterResponder("GET", url, func(req *http.Request) (*http.Response, error) { received = true @@ -721,10 +719,8 @@ func Test__GetWorkflows__WithCreationTimestampFilters(t *testing.T) { "workflows", "--project-name", "foo", - "--created-after", - threeMonthsAgo, - "--created-before", - oneMonthAgo, + "--age", + age.String(), }) RootCmd.Execute() From 454c2c10c323cfce74595d71c69e88e2a2ab9989 Mon Sep 17 00:00:00 2001 From: Lucas Pinheiro Date: Tue, 8 Oct 2024 14:16:50 -0300 Subject: [PATCH 4/4] flag description --- cmd/get.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/get.go b/cmd/get.go index c61e3c5..ba56deb 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -549,7 +549,7 @@ func init() { GetPplCmd.Flags().StringP("project-id", "i", "", "project id; if not specified will be inferred from git origin") GetPplCmd.Flags().DurationP("age", "", DefaultListingAge, - "filter for listing pipelines based on age; by default, we list only pipelines created in the last 90 days") + "list only pipelines created in the given duration; it accepts a Go duration. e.g. 24h, 30m, 60s") getCmd.AddCommand(GetPplCmd) getCmd.AddCommand(GetWfCmd) @@ -558,7 +558,7 @@ func init() { GetWfCmd.Flags().StringP("project-id", "i", "", "project id; if not specified will be inferred from git origin") GetWfCmd.Flags().DurationP("age", "", DefaultListingAge, - "filter for listing pipelines based on age; by default, we list only pipelines created in the last 90 days") + "list only workflows created in the given duration; it accepts a Go duration. e.g. 24h, 30m, 60s") getCmd.AddCommand(GetDTCmd) GetDTCmd.Flags().StringP("project-name", "p", "",