diff --git a/go/pkg/yolosvc/driver_github.go b/go/pkg/yolosvc/driver_github.go index 0688dccc..117f2f72 100644 --- a/go/pkg/yolosvc/driver_github.go +++ b/go/pkg/yolosvc/driver_github.go @@ -64,6 +64,7 @@ func (svc *service) GitHubWorker(ctx context.Context, opts GithubWorkerOpts) err worker.logger.Debug("refresh", zap.Int("iteration", iteration)) } + // fetch repo activity for _, repo := range worker.repoConfigs { // FIXME: support "since" batch, err := worker.fetchRepoActivity(ctx, repo) @@ -76,6 +77,21 @@ func (svc *service) GitHubWorker(ctx context.Context, opts GithubWorkerOpts) err } } + // maintenance: try to fix missing links + if iteration%10 == 0 { + for _, repo := range worker.repoConfigs { + // FIXME: support "since" + batch, err := worker.repoMaintenance(ctx, repo) + if err != nil { + worker.logger.Warn("fetch", zap.Error(err)) + } else { + if err := svc.saveBatch(ctx, batch); err != nil { + worker.logger.Warn("save batch", zap.Error(err)) + } + } + } + } + // FIXME: subscribe to orgs' events limits, _, err := svc.ghc.RateLimits(ctx) @@ -144,7 +160,7 @@ func (worker *githubWorker) fetchBaseObjects(ctx context.Context) (*yolopb.Batch return nil, err } logger.Debug("github.Actions.ListWorkflows", zap.Int("total", len(workflows.Workflows)), zap.Duration("duration", time.Since(before))) - fmt.Println(godev.PrettyJSON(workflows)) + fmt.Println(u.PrettyJSON(workflows)) } */ @@ -159,6 +175,43 @@ func (worker *githubWorker) fetchBaseObjects(ctx context.Context) (*yolopb.Batch return batch, nil } +func (worker *githubWorker) repoMaintenance(ctx context.Context, repo githubRepoConfig) (*yolopb.Batch, error) { + batch := yolopb.NewBatch() + + // github builds without PR + { + var builds []*yolopb.Build + err := worker.svc.db. + Where(yolopb.Build{Driver: yolopb.Driver_GitHub}). + Where("has_mergerequest_id IS NULL OR has_mergerequest_id = ''"). + Find(&builds). + Error + if err != nil { + return nil, err + } + worker.logger.Debug("repo maintenance", zap.String("repo", repo.repo), zap.Int("orphan builds", len(builds))) + // FIXME: parallelize? + for _, build := range builds { + before := time.Now() + opts := &github.PullRequestListOptions{} + ret, _, err := worker.svc.ghc.PullRequests.ListPullRequestsWithCommit(ctx, repo.owner, repo.repo, build.HasCommitID, opts) + if err != nil { + return nil, err + } + worker.logger.Debug("github.PullRequests.ListPulLRequestsWithCommit", + zap.Int("total", len(ret)), + zap.Duration("duration", time.Since(before)), + ) + if len(ret) == 1 { + build.HasMergerequestID = ret[0].GetHTMLURL() + batch.Builds = append(batch.Builds, build) + } + } + } + + return batch, nil +} + func (worker *githubWorker) fetchRepoActivity(ctx context.Context, repo githubRepoConfig) (*yolopb.Batch, error) { batch := yolopb.NewBatch() @@ -195,6 +248,7 @@ func (worker *githubWorker) fetchRepoActivity(ctx context.Context, repo githubRe runs = ret.WorkflowRuns worker.logger.Debug("github.Actions.ListRepositoryWorkflowRuns", zap.Int("total", len(runs)), zap.Duration("duration", time.Since(before))) + // FIXME: parallelize? for _, run := range runs { // fetch PRs associated to this run if run.GetHeadBranch() != "master" { @@ -213,6 +267,7 @@ func (worker *githubWorker) fetchRepoActivity(ctx context.Context, repo githubRe // fetch artifacts // FIXME: try to make a single call that gets artifacts for a repo with link to builds { + // FIXME: parallelize? for _, run := range runs { if run.GetStatus() != "completed" { continue @@ -328,7 +383,7 @@ func (worker *githubWorker) batchFromWorkflowRun(run *github.WorkflowRun, prs [] } if len(prs) > 0 { - pr := prs[0] // only take first + pr := prs[0] // only take the first one newBuild.HasMergerequestID = pr.GetHTMLURL() } diff --git a/go/pkg/yolosvc/server.go b/go/pkg/yolosvc/server.go index 641d156a..4a4eb1d8 100644 --- a/go/pkg/yolosvc/server.go +++ b/go/pkg/yolosvc/server.go @@ -3,7 +3,6 @@ package yolosvc import ( "context" "crypto/subtle" - "encoding/json" "fmt" "net" "net/http" @@ -30,6 +29,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "moul.io/u" ) type Server struct { @@ -243,8 +243,7 @@ func httpError(w http.ResponseWriter, err error, code codes.Code) { Message: code.String(), Details: fmt.Sprintf("%v", err), } - out, _ := json.MarshalIndent(msg, "", " ") - http.Error(w, string(out), runtime.HTTPStatusFromCode(code)) + http.Error(w, u.PrettyJSON(msg), runtime.HTTPStatusFromCode(code)) } func (o *ServerOpts) applyDefaults() {