feat(yolo.json): add handling of a yolo.json file to override build's…

… infos

Signed-off-by: ismael FALL <[email protected]>

api/yolopb.proto

@@ -120,6 +120,12 @@ message BuildListFilters {
 // DB Schemas
 //
 
+message overrideBuild {
+  string branch = 1;
+  string has_commit_id = 2 [(gogoproto.customname) = "HasCommitID"];
+  string has_project_id = 3 [(gogoproto.customname) = "HasProjectID"];
+}
+
 message Build {
   /// fields
 
@@ -141,6 +147,10 @@ message Build {
 
   /// relationships
 
+  string raw_branch = 21;
+  string has_raw_commit_id = 22 [(gogoproto.customname) = "HasRawCommitID"];
+  string has_raw_project_id = 23 [(gogoproto.customname) = "HasRawProjectID"];
+
   repeated Artifact has_artifacts = 101 [(gogoproto.moretags) = "gorm:\"foreignkey:HasBuildID\""];
   Commit has_commit = 102;
   string has_commit_id = 103 [(gogoproto.customname) = "HasCommitID"];

go/cmd/yolo/main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"berty.tech/yolo/v2/go/pkg/yolotree"
 	"context"
 	"flag"
 	"fmt"
@@ -13,6 +12,8 @@ import (
 	"syscall"
 	"time"
 
+	"berty.tech/yolo/v2/go/pkg/yolotree"
+
 	"berty.tech/yolo/v2/go/pkg/bintray"
 	"berty.tech/yolo/v2/go/pkg/yolopb"
 	"berty.tech/yolo/v2/go/pkg/yolosvc"
@@ -171,7 +172,7 @@ func yolo(args []string) error {
 			}
 
 			if artifactsCachePath != "" {
-				if err := os.MkdirAll(artifactsCachePath, 0755); err != nil {
+				if err := os.MkdirAll(artifactsCachePath, 0o755); err != nil {
 					return err
 				}
 			}
@@ -212,7 +213,7 @@ func yolo(args []string) error {
 				gr.Add(func() error { return svc.PkgmanWorker(ctx, opts) }, func(_ error) { cancel() })
 			}
 			if githubToken != "" {
-				opts := yolosvc.GithubWorkerOpts{Logger: logger, MaxBuilds: maxBuilds, ClearCache: cc, Once: once, ReposFilter: githubRepos}
+				opts := yolosvc.GithubWorkerOpts{Logger: logger, MaxBuilds: maxBuilds, ClearCache: cc, Once: once, ReposFilter: githubRepos, Token: githubToken}
 				gr.Add(func() error { return svc.GitHubWorker(ctx, opts) }, func(_ error) { cancel() })
 			}
 

go/pkg/yolosvc/driver_github.go

@@ -1,11 +1,18 @@
 package yolosvc
 
 import (
+	"archive/zip"
+	"bytes"
 	"context"
+	"errors"
 	"fmt"
+	"io/ioutil"
+	"net/http"
 	"strings"
 	"time"
 
+	"github.com/golang/protobuf/jsonpb"
+
 	"berty.tech/yolo/v2/go/pkg/yolopb"
 	"github.com/google/go-github/v32/github"
 	"github.com/tevino/abool"
@@ -19,6 +26,7 @@ type GithubWorkerOpts struct {
 	ClearCache  *abool.AtomicBool
 	Once        bool
 	ReposFilter string
+	Token       string
 }
 
 type githubRepoConfig struct {
@@ -34,7 +42,7 @@ type githubWorker struct {
 	repoConfigs []githubRepoConfig
 }
 
-func (worker *githubWorker) ParseConfig() (bool, error) {
+func (worker *githubWorker) parseConfig() (bool, error) {
 	if worker.opts.ReposFilter == "" {
 		return false, nil
 	}
@@ -66,7 +74,7 @@ func (svc *service) GitHubWorker(ctx context.Context, opts GithubWorkerOpts) err
 		logger: opts.Logger.Named("ghub"),
 	}
 
-	shouldRun, err := worker.ParseConfig()
+	shouldRun, err := worker.parseConfig()
 	if err != nil {
 		svc.logger.Error("invalid config", zap.Error(err))
 		return fmt.Errorf("%w", err)
@@ -169,6 +177,48 @@ func (worker *githubWorker) fetchBaseObjects(ctx context.Context) (*yolopb.Batch
 	return batch, nil
 }
 
+func getOverrideBuildpb(owner string, repo string, artiID int64, token string) (*yolopb.OverrideBuild, error) {
+	override := &yolopb.OverrideBuild{}
+	// FIXME: is it always .zip files ?
+	req, err := http.NewRequest("GET", fmt.Sprintf("https://api.github.com/repos/%s/%s/actions/artifacts/%d/zip", owner, repo, artiID), nil)
+	if err != nil {
+		return nil, errors.New("getOverrideBuildpb: " + err.Error())
+	}
+
+	req.Header.Set("Accept", "application/vnd.github+json")
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
+
+	do, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, errors.New("getOverrideBuildpb: " + err.Error())
+	}
+	body, err := ioutil.ReadAll(do.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
+	if err != nil {
+		return nil, errors.New("getOverrideBuildpb: " + err.Error())
+	}
+
+	for _, zipFile := range zipReader.File {
+		if zipFile.Name == "yolo.json" {
+			unzippedFileBytes, err := readZipFile(zipFile)
+			if err != nil {
+				return nil, errors.New("getOverrideBuildpb: " + err.Error())
+			}
+
+			err = jsonpb.Unmarshal(bytes.NewReader(unzippedFileBytes), override)
+			if err != nil {
+				return nil, errors.New("getOverrideBuildpb: " + err.Error())
+			}
+			return override, nil
+		}
+	}
+	return nil, nil
+}
+
 func (worker *githubWorker) fetchRepoActivity(ctx context.Context, repo githubRepoConfig, iteration int, lastFinishedBuild time.Time) (*yolopb.Batch, error) {
 	batch := yolopb.NewBatch()
 
@@ -222,6 +272,7 @@ func (worker *githubWorker) fetchRepoActivity(ctx context.Context, repo githubRe
 			if err != nil {
 				return nil, err
 			}
+
 			worker.logger.Debug("github.Actions.ListRepositoryWorkflowRuns",
 				zap.Int("total", len(ret.WorkflowRuns)),
 				zap.Duration("duration", time.Since(before)),
@@ -231,28 +282,45 @@ func (worker *githubWorker) fetchRepoActivity(ctx context.Context, repo githubRe
 
 			// FIXME: parallelize?
 			for _, run := range ret.WorkflowRuns {
-				// FIXME: compare updated_at before doing next calls
-				runs = append(runs, run)
-
-				isFork := run.GetHeadRepository().GetOwner().GetLogin() != repo.owner ||
-					run.GetHeadRepository().GetName() != repo.repo
-
-				// fetch PRs associated to this run
-				if isFork || run.GetHeadBranch() != "master" {
-					opts := &github.PullRequestListOptions{}
-					ret, _, err := worker.svc.ghc.PullRequests.ListPullRequestsWithCommit(
-						ctx,
-						run.GetHeadRepository().GetOwner().GetLogin(),
-						run.GetHeadRepository().GetName(),
-						run.GetHeadSHA(),
-						opts,
-					)
-					if err != nil {
-						return nil, err
+				var overridepb *yolopb.OverrideBuild
+				// check for yolo.json
+				opts := &github.ListOptions{}
+				retArti, _, err := worker.svc.ghc.Actions.ListWorkflowRunArtifacts(ctx, repo.owner, repo.repo, *run.ID, opts)
+				if err != nil {
+					return nil, err
+				}
+				for _, arti := range retArti.Artifacts {
+					if *arti.Name == "yolo.json" {
+						// download the override config
+						overridepb, err = getOverrideBuildpb(repo.owner, repo.repo, *arti.ID, worker.opts.Token)
+						if err != nil {
+							return nil, err
+						}
+					}
+
+					// FIXME: compare updated_at before doing next calls
+					runs = append(runs, run)
+
+					isFork := run.GetHeadRepository().GetOwner().GetLogin() != repo.owner ||
+						run.GetHeadRepository().GetName() != repo.repo
+
+					// fetch PRs associated to this run
+					if isFork || run.GetHeadBranch() != "master" {
+						opts := &github.PullRequestListOptions{}
+						ret, _, err := worker.svc.ghc.PullRequests.ListPullRequestsWithCommit(
+							ctx,
+							run.GetHeadRepository().GetOwner().GetLogin(),
+							run.GetHeadRepository().GetName(),
+							run.GetHeadSHA(),
+							opts,
+						)
+						if err != nil {
+							return nil, err
+						}
+						batch.Merge(worker.batchFromWorkflowRun(run, ret, overridepb))
+					} else {
+						batch.Merge(worker.batchFromWorkflowRun(run, nil, overridepb))
 					}
-					batch.Merge(worker.batchFromWorkflowRun(run, ret))
-				} else {
-					batch.Merge(worker.batchFromWorkflowRun(run, nil))
 				}
 			}
 		}
@@ -327,23 +395,41 @@ func (worker *githubWorker) batchFromWorkflowRunArtifact(run *github.WorkflowRun
 	return batch
 }
 
-func (worker *githubWorker) batchFromWorkflowRun(run *github.WorkflowRun, prs []*github.PullRequest) *yolopb.Batch {
+func (worker *githubWorker) batchFromWorkflowRun(run *github.WorkflowRun, prs []*github.PullRequest, overrideRun *yolopb.OverrideBuild) *yolopb.Batch {
 	batch := yolopb.NewBatch()
 	createdAt := run.GetCreatedAt().Time
 	updatedAt := run.GetUpdatedAt().Time
 	commitURL := run.GetHeadRepository().GetHTMLURL() + "/commit/" + run.GetHeadSHA()
 
 	newBuild := yolopb.Build{
-		ID:           run.GetHTMLURL(),
-		ShortID:      fmt.Sprintf("%d", run.GetID()),
-		CreatedAt:    &createdAt,
-		UpdatedAt:    &updatedAt,
-		HasCommitID:  run.GetHeadSHA(),
-		Branch:       run.GetHeadBranch(),
-		Driver:       yolopb.Driver_GitHub,
-		CommitURL:    commitURL,
-		HasProjectID: run.GetRepository().GetHTMLURL(),
-		Message:      run.GetHeadCommit().GetMessage(),
+		ID:              run.GetHTMLURL(),
+		ShortID:         fmt.Sprintf("%d", run.GetID()),
+		CreatedAt:       &createdAt,
+		UpdatedAt:       &updatedAt,
+		HasRawCommitID:  run.GetHeadSHA(),
+		HasCommitID:     run.GetHeadSHA(),
+		RawBranch:       run.GetHeadBranch(),
+		Branch:          run.GetHeadBranch(),
+		Driver:          yolopb.Driver_GitHub,
+		CommitURL:       commitURL,
+		HasRawProjectID: run.GetRepository().GetHTMLURL(),
+		HasProjectID:    run.GetRepository().GetHTMLURL(),
+		Message:         run.GetHeadCommit().GetMessage(),
+	}
+
+	if overrideRun != nil {
+		// override project ID
+		if overrideRun.HasProjectID != "" {
+			newBuild.HasProjectID = overrideRun.HasProjectID
+		}
+		// override commit ID
+		if overrideRun.HasCommitID != "" {
+			newBuild.HasCommitID = overrideRun.HasCommitID
+		}
+		// override branch
+		if overrideRun.Branch != "" {
+			newBuild.Branch = overrideRun.Branch
+		}
 	}
 
 	if run.GetConclusion() != "" {
@@ -374,9 +460,9 @@ func (worker *githubWorker) batchFromWorkflowRun(run *github.WorkflowRun, prs []
 			newBuild.State = yolopb.Build_Skipped
 		case conclusion == "timed_out":
 			newBuild.State = yolopb.Build_Timedout
-		//case conclusion == "action_required":
-		//case conclusion == "neutral":
-		//case conclusion == "state":
+		// case conclusion == "action_required":
+		// case conclusion == "neutral":
+		// case conclusion == "state":
 		default:
 			newBuild.State = yolopb.Build_UnknownState
 			worker.logger.Error(

go/pkg/yolosvc/util.go

@@ -1,24 +1,24 @@
 package yolosvc
 
 import (
+	"archive/zip"
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
+	"io/ioutil"
 	"path/filepath"
 	"regexp"
 
 	"berty.tech/yolo/v2/go/pkg/yolopb"
 )
 
-var (
-	githubMasterMerge = regexp.MustCompile(`Merge pull request #([0-9]+) from (.*)`)
-)
+var githubMasterMerge = regexp.MustCompile(`Merge pull request #([0-9]+) from (.*)`)
 
 func artifactKindByPath(path string) yolopb.Artifact_Kind {
 	switch filepath.Ext(path) {
 	case ".ipa", ".unsigned-ipa", ".dummy-signed-ipa":
 		return yolopb.Artifact_IPA
-	case ".dmg", ".unsigned-dmg" , ".dummy-signed-dmg":
+	case ".dmg", ".unsigned-dmg", ".dummy-signed-dmg":
 		return yolopb.Artifact_DMG
 	case ".apk":
 		return yolopb.Artifact_APK
@@ -69,3 +69,12 @@ func guessMissingBuildInfo(build *yolopb.Build) {
 		build.VCSTagURL = fmt.Sprintf("%s/tree/%s", build.HasProjectID, build.VCSTag)
 	}
 }
+
+func readZipFile(zf *zip.File) ([]byte, error) {
+	f, err := zf.Open()
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return ioutil.ReadAll(f)
+}

go/pkg/yolotree/yolotree.go

@@ -1,9 +1,10 @@
 package yolotree
 
 import (
-	"berty.tech/yolo/v2/go/pkg/yolopb"
 	"fmt"
 	"sort"
+
+	"berty.tech/yolo/v2/go/pkg/yolopb"
 )
 
 type artifact struct {