From 4bcab65133a96fefaa7072c21a9c8a9c4ccd53ea Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Thu, 22 Sep 2022 10:19:13 -0500 Subject: [PATCH] Add support for git commit hashes (#755) * Add support for git commit hashes * Fixup example docs and comments * Resolve Jon feedback * Address JVB concerns Co-authored-by: Megamind <882485+jeff-mccoy@users.noreply.github.com> --- examples/git-data/README.md | 16 ++++++++-- examples/git-data/zarf.yaml | 6 ++-- src/internal/git/checkout.go | 4 +-- src/internal/git/fetch.go | 46 +++++++++++++++++++--------- src/internal/git/pull.go | 24 ++++++++++----- src/internal/git/push.go | 23 +++++++++++++- src/internal/packager/deploy.go | 2 +- src/test/e2e/22_git_and_flux_test.go | 39 ++++++++++++++++++++--- 8 files changed, 126 insertions(+), 34 deletions(-) diff --git a/examples/git-data/README.md b/examples/git-data/README.md index 2aea1df7ec..ba687af35b 100644 --- a/examples/git-data/README.md +++ b/examples/git-data/README.md @@ -13,9 +13,21 @@ To view the example source code, select the `Edit this page` link below the arti ## Tag-Provided Git Repository Clone -Tag-provided `git` repository cloning is the recommended way of cloning a `git` repository for air-gapped deployments. Tag-provided clones are defined using the `url.git@tag` format as seen in the example of the `defenseunicorns/zarf` repository (`https://github.com/defenseunicorns/zarf.git@v0.12.0`). +Tag-provided `git` repository cloning is the recommended way of cloning a `git` repository for air-gapped deployments because it wraps meaning around a specific point in git history that can easily be traced back to the online world. Tag-provided clones are defined using the `url.git@tag` format as seen in the example of the `defenseunicorns/zarf` repository (`https://github.com/defenseunicorns/zarf.git@v0.15.0`). -A tag-provided clone only mirrors the tag defined in the Zarf definition. The tag will appear on the `git` mirror as the default branch name of the repository being mirrored, along with the tag itself. +A tag-provided clone only mirrors the tag defined in the Zarf definition. The tag will be applied on the `git` mirror to the default trunk branch of the repo (i.e. `master`, `main`, or the default when the repo is cloned). + +## SHA-Provided Git Repository Clone + +SHA-provided `git` repository cloning is another supported way of cloning repos in Zarf but is not recommended as it is less readable / understandable than tag cloning. Commit SHAs are defined using the same `url.git@sha` format as seen in the example of the `defenseunicorns/zarf` repository (`https://github.com/defenseunicorns/zarf.git@c74e2e9626da0400e0a41e78319b3054c53a5d4e`). + +A SHA-provided clone only mirrors the SHA hash defined in the Zarf definition. The SHA will be applied on the `git` mirror to the default trunk branch of the repo (i.e. `master`, `main`, or the default when the repo is cloned) as Zarf does with tagging. + +:::note + +If you use a SHA hash or a tag that is on a separate branch this will be placed on the default trunk branch on the offline mirror (i.e. `master`, `main`, etc). This may result in conflicts upon updates if this SHA or tag is not in the update branch's history. + +::: ## Git Repository Full Clone diff --git a/examples/git-data/zarf.yaml b/examples/git-data/zarf.yaml index 631560fba3..c2f98cfcc6 100644 --- a/examples/git-data/zarf.yaml +++ b/examples/git-data/zarf.yaml @@ -9,9 +9,11 @@ components: images: - ghcr.io/stefanprodan/podinfo:6.0.0 repos: - # Do a tag-provided Git Repo mirror + # Do a tag-provided Git Repo mirror - https://github.com/defenseunicorns/zarf.git@v0.15.0 - # Do a tag-provided Git Repo mirror with the default branch of main + # Do a commit hash Git Repo mirror + - https://github.com/defenseunicorns/zarf.git@c74e2e9626da0400e0a41e78319b3054c53a5d4e + # Do a tag-provided Git Repo mirror with the default branch of main - https://repo1.dso.mil/platform-one/big-bang/apps/security-tools/twistlock.git@0.0.9-bb.0 # Do a full Git Repo Mirror - https://github.com/stefanprodan/podinfo.git diff --git a/src/internal/git/checkout.go b/src/internal/git/checkout.go index d9c4804556..2a16e85717 100644 --- a/src/internal/git/checkout.go +++ b/src/internal/git/checkout.go @@ -15,10 +15,10 @@ func CheckoutTag(path string, tag string) { checkout(path, options) } -// CheckoutTagAsBranch performs a `git checkout` of the provided tag but rather +// checkoutTagAsBranch performs a `git checkout` of the provided tag but rather // than checking out to a detached head, checks out to the provided branch ref // It will delete the branch provided if it exists -func CheckoutTagAsBranch(path string, tag string, branch plumbing.ReferenceName) { +func checkoutTagAsBranch(path string, tag string, branch plumbing.ReferenceName) { message.Debugf("Checkout tag %s as branch %s for %s", tag, branch.String(), path) repo, err := git.PlainOpen(path) if err != nil { diff --git a/src/internal/git/fetch.go b/src/internal/git/fetch.go index fa6163d1f3..204fe54205 100644 --- a/src/internal/git/fetch.go +++ b/src/internal/git/fetch.go @@ -8,10 +8,36 @@ import ( goConfig "github.com/go-git/go-git/v5/config" ) -// fetchTag performs a `git fetch` of _only_ the provided tag +// fetchTag performs a `git fetch` of _only_ the provided tag. func fetchTag(gitDirectory string, tag string) { message.Debugf("Fetch git tag %s from repo %s", tag, path.Base(gitDirectory)) + refspec := goConfig.RefSpec("refs/tags/" + tag + ":refs/tags/" + tag) + + err := fetch(gitDirectory, refspec) + + if err == git.ErrTagExists { + message.Debug("Tag already fetched") + } else if err != nil { + message.Fatal(err, "Not a valid tag or unable to fetch") + } +} + +// fetchHash performs a `git fetch` of _only_ the provided commit hash. +func fetchHash(gitDirectory string, hash string) { + message.Debugf("Fetch git hash %s from repo %s", hash, path.Base(gitDirectory)) + + refspec := goConfig.RefSpec(hash + ":" + hash) + + err := fetch(gitDirectory, refspec) + + if err != nil { + message.Fatal(err, "Not a valid hash or unable to fetch") + } +} + +// fetch performs a `git fetch` of _only_ the provided git refspec. +func fetch(gitDirectory string, refspec goConfig.RefSpec) error { repo, err := git.PlainOpen(gitDirectory) if err != nil { message.Fatal(err, "Unable to load the git repo") @@ -24,27 +50,19 @@ func fetchTag(gitDirectory string, tag string) { message.Fatal(err, "Failed to identify remotes.") } - gitUrl := remotes[0].Config().URLs[0] - message.Debugf("Attempting to find tag: %s for %s", tag, gitUrl) + gitURL := remotes[0].Config().URLs[0] + message.Debugf("Attempting to find ref: %s for %s", refspec.String(), gitURL) - gitCred := FindAuthForHost(gitUrl) + gitCred := FindAuthForHost(gitURL) fetchOptions := &git.FetchOptions{ RemoteName: onlineRemoteName, - RefSpecs: []goConfig.RefSpec{ - goConfig.RefSpec("refs/tags/" + tag + ":refs/tags/" + tag), - }, + RefSpecs: []goConfig.RefSpec{refspec}, } if gitCred.Auth.Username != "" { fetchOptions.Auth = &gitCred.Auth } - err = repo.Fetch(fetchOptions) - - if err == git.ErrTagExists { - message.Debug("Tag already fetched") - } else if err != nil { - message.Fatal(err, "Not a valid tag or unable to fetch") - } + return repo.Fetch(fetchOptions) } diff --git a/src/internal/git/pull.go b/src/internal/git/pull.go index 3c910bbb25..bf2c9c1dc2 100644 --- a/src/internal/git/pull.go +++ b/src/internal/git/pull.go @@ -2,6 +2,7 @@ package git import ( "context" + "regexp" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/message" @@ -35,14 +36,14 @@ func pull(gitUrl, targetFolder string, spinner *message.Spinner) { gitCred := FindAuthForHost(gitUrl) matches := strings.Split(gitUrl, "@") - fetchAllTags := len(matches) == 1 + onlyFetchRef := len(matches) == 2 cloneOptions := &git.CloneOptions{ URL: matches[0], Progress: spinner, RemoteName: onlineRemoteName, } - if !fetchAllTags { + if onlyFetchRef { cloneOptions.Tags = git.NoTags } @@ -73,8 +74,8 @@ func pull(gitUrl, targetFolder string, spinner *message.Spinner) { return } - if !fetchAllTags { - tag := matches[1] + if onlyFetchRef { + ref := matches[1] // Identify the remote trunk branch name trunkBranchName := plumbing.NewBranchReferenceName("master") @@ -82,19 +83,26 @@ func pull(gitUrl, targetFolder string, spinner *message.Spinner) { if err != nil { // No repo head available - spinner.Errorf(err, "Failed to identify repo head. Tag will be pushed to 'master'.") + spinner.Errorf(err, "Failed to identify repo head. Ref will be pushed to 'master'.") } else if head.Name().IsBranch() { // Valid repo head and it is a branch trunkBranchName = head.Name() } else { // Valid repo head but not a branch - spinner.Errorf(nil, "No branch found for this repo head. Tag will be pushed to 'master'.") + spinner.Errorf(nil, "No branch found for this repo head. Ref will be pushed to 'master'.") } _, _ = removeLocalBranchRefs(targetFolder) _, _ = removeOnlineRemoteRefs(targetFolder) - fetchTag(targetFolder, tag) - CheckoutTagAsBranch(targetFolder, tag, trunkBranchName) + var isHash = regexp.MustCompile(`^[0-9a-f]{40}$`).MatchString + + if isHash(ref) { + fetchHash(targetFolder, ref) + checkoutHashAsBranch(targetFolder, plumbing.NewHash(ref), trunkBranchName) + } else { + fetchTag(targetFolder, ref) + checkoutTagAsBranch(targetFolder, ref, trunkBranchName) + } } } diff --git a/src/internal/git/push.go b/src/internal/git/push.go index c938dec001..89b4d73459 100644 --- a/src/internal/git/push.go +++ b/src/internal/git/push.go @@ -1,6 +1,7 @@ package git import ( + "errors" "fmt" "path/filepath" "strings" @@ -11,6 +12,7 @@ import ( "github.com/defenseunicorns/zarf/src/internal/utils" "github.com/go-git/go-git/v5" goConfig "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" ) @@ -96,6 +98,25 @@ func push(localPath, tunnelUrl string, spinner *message.Spinner) error { return fmt.Errorf("unable to remove unused git refs from the repo: %w", err) } + // Fetch remote offline refs in case of old update or if multiple refs are specified in one package + fetchOptions := &git.FetchOptions{ + RemoteName: offlineRemoteName, + Auth: &gitCred, + RefSpecs: []goConfig.RefSpec{ + "refs/heads/*:refs/heads/*", + onlineRemoteRefPrefix + "*:refs/heads/*", + "refs/tags/*:refs/tags/*", + }, + } + + err = repo.Fetch(fetchOptions) + if errors.Is(err, transport.ErrRepositoryNotFound) { + message.Debugf("Repo not yet available offline, skipping fetch") + } else if err != nil { + return fmt.Errorf("unable to fetch remote cleanly prior to push: %w", err) + } + + // Push all heads and tags to the offline remote err = repo.Push(&git.PushOptions{ RemoteName: offlineRemoteName, Auth: &gitCred, @@ -108,7 +129,7 @@ func push(localPath, tunnelUrl string, spinner *message.Spinner) error { }, }) - if err == git.NoErrAlreadyUpToDate { + if errors.Is(err, git.NoErrAlreadyUpToDate) { spinner.Debugf("Repo already up-to-date") } else if err != nil { return fmt.Errorf("unable to push repo to the gitops service: %w", err) diff --git a/src/internal/packager/deploy.go b/src/internal/packager/deploy.go index 3d8d950209..8476c00a1c 100644 --- a/src/internal/packager/deploy.go +++ b/src/internal/packager/deploy.go @@ -291,7 +291,7 @@ func deployComponents(tempPath tempPaths, component types.ZarfComponent) []types for retry := 0; retry < 3; retry++ { // Push all the repos from the extracted archive if err := git.PushAllDirectories(componentPath.repos); err != nil { - message.Errorf(err, "Unable to push repos to the Zarf Registry, retrying in 5 seconds...") + message.Errorf(err, "Unable to push repos to the Zarf Repository, retrying in 5 seconds...") time.Sleep(5 * time.Second) continue } else { diff --git a/src/test/e2e/22_git_and_flux_test.go b/src/test/e2e/22_git_and_flux_test.go index a260f7c1d4..fac493deda 100644 --- a/src/test/e2e/22_git_and_flux_test.go +++ b/src/test/e2e/22_git_and_flux_test.go @@ -32,6 +32,7 @@ func TestGitAndFlux(t *testing.T) { testGitServerConnect(t, tunnel.HttpEndpoint()) testGitServerReadOnly(t, tunnel.HttpEndpoint()) + testGitServerTagAndHash(t, tunnel.HttpEndpoint()) waitFluxPodInfoDeployment(t) stdOut, stdErr, err = e2e.execZarfCommand("package", "remove", "flux-test", "--confirm") @@ -41,21 +42,21 @@ func TestGitAndFlux(t *testing.T) { require.NoError(t, err, stdOut, stdErr) } -func testGitServerConnect(t *testing.T, gitUrl string) { +func testGitServerConnect(t *testing.T, gitURL string) { // Make sure Gitea comes up cleanly - resp, err := http.Get(gitUrl + "/explore/repos") + resp, err := http.Get(gitURL + "/explore/repos") assert.NoError(t, err) assert.Equal(t, 200, resp.StatusCode) } -func testGitServerReadOnly(t *testing.T, gitUrl string) { +func testGitServerReadOnly(t *testing.T, gitURL string) { // Init the state variable state := k8s.LoadZarfState() config.InitState(state) // Get the repo as the readonly user repoName := "mirror__repo1.dso.mil__platform-one__big-bang__apps__security-tools__twistlock" - getRepoRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s", gitUrl, config.ZarfGitPushUser, repoName), nil) + getRepoRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s", gitURL, config.ZarfGitPushUser, repoName), nil) getRepoResponseBody, err := git.DoHttpThings(getRepoRequest, config.ZarfGitReadUser, config.GetSecret(config.StateGitPull)) assert.NoError(t, err) @@ -68,6 +69,36 @@ func testGitServerReadOnly(t *testing.T, gitUrl string) { assert.True(t, permissionsMap["pull"].(bool)) } +func testGitServerTagAndHash(t *testing.T, gitURL string) { + // Init the state variable + state := k8s.LoadZarfState() + config.InitState(state) + + repoName := "mirror__github.com__defenseunicorns__zarf" + + // Get the Zarf repo tag + repoTag := "v0.15.0" + getRepoTagsRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s/tags/%s", gitURL, config.ZarfGitPushUser, repoName, repoTag), nil) + getRepoTagsResponseBody, err := git.DoHttpThings(getRepoTagsRequest, config.ZarfGitReadUser, config.GetSecret(config.StateGitPull)) + assert.NoError(t, err) + + // Make sure the pushed tag exists + var tagMap map[string]interface{} + json.Unmarshal(getRepoTagsResponseBody, &tagMap) + assert.Equal(t, repoTag, tagMap["name"]) + + // Get the Zarf repo commit + repoHash := "c74e2e9626da0400e0a41e78319b3054c53a5d4e" + getRepoCommitsRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s/commits", gitURL, config.ZarfGitPushUser, repoName), nil) + getRepoCommitsResponseBody, err := git.DoHttpThings(getRepoCommitsRequest, config.ZarfGitReadUser, config.GetSecret(config.StateGitPull)) + assert.NoError(t, err) + + // Make sure the pushed commit exists + var commitMap []map[string]interface{} + json.Unmarshal(getRepoCommitsResponseBody, &commitMap) + assert.Equal(t, repoHash, commitMap[0]["sha"]) +} + func waitFluxPodInfoDeployment(t *testing.T) { // Deploy the flux example and verify that it works path := fmt.Sprintf("build/zarf-package-flux-test-%s.tar.zst", e2e.arch)