Add support for git commit hashes (#755)

* Add support for git commit hashes

* Fixup example docs and comments

* Resolve Jon feedback

* Address JVB concerns

Co-authored-by: Megamind <[email protected]>

examples/git-data/README.md

@@ -13,9 +13,21 @@ To view the example source code, select the `Edit this page` link below the arti
 
 ## Tag-Provided Git Repository Clone
 
-Tag-provided `git` repository cloning is the recommended way of cloning a `git` repository for air-gapped deployments. Tag-provided clones are defined using the `url.git@tag` format as seen in the example of the `defenseunicorns/zarf` repository (`https://github.com/defenseunicorns/zarf.git@v0.12.0`).
+Tag-provided `git` repository cloning is the recommended way of cloning a `git` repository for air-gapped deployments because it wraps meaning around a specific point in git history that can easily be traced back to the online world. Tag-provided clones are defined using the `url.git@tag` format as seen in the example of the `defenseunicorns/zarf` repository (`https://github.com/defenseunicorns/zarf.git@v0.15.0`).
 
-A tag-provided clone only mirrors the tag defined in the Zarf definition. The tag will appear on the `git` mirror as the default branch name of the repository being mirrored, along with the tag itself.
+A tag-provided clone only mirrors the tag defined in the Zarf definition. The tag will be applied on the `git` mirror to the default trunk branch of the repo (i.e. `master`, `main`, or the default when the repo is cloned).
+
+## SHA-Provided Git Repository Clone
+
+SHA-provided `git` repository cloning is another supported way of cloning repos in Zarf but is not recommended as it is less readable / understandable than tag cloning.  Commit SHAs are defined using the same `url.git@sha` format as seen in the example of the `defenseunicorns/zarf` repository (`https://github.com/defenseunicorns/zarf.git@c74e2e9626da0400e0a41e78319b3054c53a5d4e`).
+
+A SHA-provided clone only mirrors the SHA hash defined in the Zarf definition. The SHA will be applied on the `git` mirror to the default trunk branch of the repo (i.e. `master`, `main`, or the default when the repo is cloned) as Zarf does with tagging.
+
+:::note
+
+If you use a SHA hash or a tag that is on a separate branch this will be placed on the default trunk branch on the offline mirror (i.e. `master`, `main`, etc).  This may result in conflicts upon updates if this SHA or tag is not in the update branch's history.
+
+:::
 
 ## Git Repository Full Clone
 

examples/git-data/zarf.yaml

@@ -9,9 +9,11 @@ components:
     images:
       - ghcr.io/stefanprodan/podinfo:6.0.0
     repos:
-      # Do a tag-provided Git Repo mirror
+        # Do a tag-provided Git Repo mirror
       - https://github.com/defenseunicorns/[email protected]
-      # Do a tag-provided Git Repo mirror with the default branch of main
+        # Do a commit hash Git Repo mirror
+      - https://github.com/defenseunicorns/zarf.git@c74e2e9626da0400e0a41e78319b3054c53a5d4e
+        # Do a tag-provided Git Repo mirror with the default branch of main
       - https://repo1.dso.mil/platform-one/big-bang/apps/security-tools/[email protected]
         # Do a full Git Repo Mirror
       - https://github.com/stefanprodan/podinfo.git

src/internal/git/checkout.go

@@ -15,10 +15,10 @@ func CheckoutTag(path string, tag string) {
 	checkout(path, options)
 }
 
-// CheckoutTagAsBranch performs a `git checkout` of the provided tag but rather
+// checkoutTagAsBranch performs a `git checkout` of the provided tag but rather
 // than checking out to a detached head, checks out to the provided branch ref
 // It will delete the branch provided if it exists
-func CheckoutTagAsBranch(path string, tag string, branch plumbing.ReferenceName) {
+func checkoutTagAsBranch(path string, tag string, branch plumbing.ReferenceName) {
 	message.Debugf("Checkout tag %s as branch %s for %s", tag, branch.String(), path)
 	repo, err := git.PlainOpen(path)
 	if err != nil {

src/internal/git/fetch.go

@@ -8,10 +8,36 @@ import (
 	goConfig "github.com/go-git/go-git/v5/config"
 )
 
-// fetchTag performs a `git fetch` of _only_ the provided tag
+// fetchTag performs a `git fetch` of _only_ the provided tag.
 func fetchTag(gitDirectory string, tag string) {
 	message.Debugf("Fetch git tag %s from repo %s", tag, path.Base(gitDirectory))
 
+	refspec := goConfig.RefSpec("refs/tags/" + tag + ":refs/tags/" + tag)
+
+	err := fetch(gitDirectory, refspec)
+
+	if err == git.ErrTagExists {
+		message.Debug("Tag already fetched")
+	} else if err != nil {
+		message.Fatal(err, "Not a valid tag or unable to fetch")
+	}
+}
+
+// fetchHash performs a `git fetch` of _only_ the provided commit hash.
+func fetchHash(gitDirectory string, hash string) {
+	message.Debugf("Fetch git hash %s from repo %s", hash, path.Base(gitDirectory))
+
+	refspec := goConfig.RefSpec(hash + ":" + hash)
+
+	err := fetch(gitDirectory, refspec)
+
+	if err != nil {
+		message.Fatal(err, "Not a valid hash or unable to fetch")
+	}
+}
+
+// fetch performs a `git fetch` of _only_ the provided git refspec.
+func fetch(gitDirectory string, refspec goConfig.RefSpec) error {
 	repo, err := git.PlainOpen(gitDirectory)
 	if err != nil {
 		message.Fatal(err, "Unable to load the git repo")
@@ -24,27 +50,19 @@ func fetchTag(gitDirectory string, tag string) {
 		message.Fatal(err, "Failed to identify remotes.")
 	}
 
-	gitUrl := remotes[0].Config().URLs[0]
-	message.Debugf("Attempting to find tag: %s for %s", tag, gitUrl)
+	gitURL := remotes[0].Config().URLs[0]
+	message.Debugf("Attempting to find ref: %s for %s", refspec.String(), gitURL)
 
-	gitCred := FindAuthForHost(gitUrl)
+	gitCred := FindAuthForHost(gitURL)
 
 	fetchOptions := &git.FetchOptions{
 		RemoteName: onlineRemoteName,
-		RefSpecs: []goConfig.RefSpec{
-			goConfig.RefSpec("refs/tags/" + tag + ":refs/tags/" + tag),
-		},
+		RefSpecs:   []goConfig.RefSpec{refspec},
 	}
 
 	if gitCred.Auth.Username != "" {
 		fetchOptions.Auth = &gitCred.Auth
 	}
 
-	err = repo.Fetch(fetchOptions)
-
-	if err == git.ErrTagExists {
-		message.Debug("Tag already fetched")
-	} else if err != nil {
-		message.Fatal(err, "Not a valid tag or unable to fetch")
-	}
+	return repo.Fetch(fetchOptions)
 }

src/internal/git/pull.go

@@ -2,6 +2,7 @@ package git
 
 import (
 	"context"
+	"regexp"
 
 	"github.com/defenseunicorns/zarf/src/config"
 	"github.com/defenseunicorns/zarf/src/internal/message"
@@ -35,14 +36,14 @@ func pull(gitUrl, targetFolder string, spinner *message.Spinner) {
 	gitCred := FindAuthForHost(gitUrl)
 
 	matches := strings.Split(gitUrl, "@")
-	fetchAllTags := len(matches) == 1
+	onlyFetchRef := len(matches) == 2
 	cloneOptions := &git.CloneOptions{
 		URL:        matches[0],
 		Progress:   spinner,
 		RemoteName: onlineRemoteName,
 	}
 
-	if !fetchAllTags {
+	if onlyFetchRef {
 		cloneOptions.Tags = git.NoTags
 	}
 
@@ -73,28 +74,35 @@ func pull(gitUrl, targetFolder string, spinner *message.Spinner) {
 		return
 	}
 
-	if !fetchAllTags {
-		tag := matches[1]
+	if onlyFetchRef {
+		ref := matches[1]
 
 		// Identify the remote trunk branch name
 		trunkBranchName := plumbing.NewBranchReferenceName("master")
 		head, err := repo.Head()
 
 		if err != nil {
 			// No repo head available
-			spinner.Errorf(err, "Failed to identify repo head. Tag will be pushed to 'master'.")
+			spinner.Errorf(err, "Failed to identify repo head. Ref will be pushed to 'master'.")
 		} else if head.Name().IsBranch() {
 			// Valid repo head and it is a branch
 			trunkBranchName = head.Name()
 		} else {
 			// Valid repo head but not a branch
-			spinner.Errorf(nil, "No branch found for this repo head. Tag will be pushed to 'master'.")
+			spinner.Errorf(nil, "No branch found for this repo head. Ref will be pushed to 'master'.")
 		}
 
 		_, _ = removeLocalBranchRefs(targetFolder)
 		_, _ = removeOnlineRemoteRefs(targetFolder)
 
-		fetchTag(targetFolder, tag)
-		CheckoutTagAsBranch(targetFolder, tag, trunkBranchName)
+		var isHash = regexp.MustCompile(`^[0-9a-f]{40}$`).MatchString
+
+		if isHash(ref) {
+			fetchHash(targetFolder, ref)
+			checkoutHashAsBranch(targetFolder, plumbing.NewHash(ref), trunkBranchName)
+		} else {
+			fetchTag(targetFolder, ref)
+			checkoutTagAsBranch(targetFolder, ref, trunkBranchName)
+		}
 	}
 }

src/internal/git/push.go

@@ -1,6 +1,7 @@
 package git
 
 import (
+	"errors"
 	"fmt"
 	"path/filepath"
 	"strings"
@@ -11,6 +12,7 @@ import (
 	"github.com/defenseunicorns/zarf/src/internal/utils"
 	"github.com/go-git/go-git/v5"
 	goConfig "github.com/go-git/go-git/v5/config"
+	"github.com/go-git/go-git/v5/plumbing/transport"
 	"github.com/go-git/go-git/v5/plumbing/transport/http"
 )
 
@@ -96,6 +98,25 @@ func push(localPath, tunnelUrl string, spinner *message.Spinner) error {
 		return fmt.Errorf("unable to remove unused git refs from the repo: %w", err)
 	}
 
+	// Fetch remote offline refs in case of old update or if multiple refs are specified in one package
+	fetchOptions := &git.FetchOptions{
+		RemoteName: offlineRemoteName,
+		Auth:       &gitCred,
+		RefSpecs: []goConfig.RefSpec{
+			"refs/heads/*:refs/heads/*",
+			onlineRemoteRefPrefix + "*:refs/heads/*",
+			"refs/tags/*:refs/tags/*",
+		},
+	}
+
+	err = repo.Fetch(fetchOptions)
+	if errors.Is(err, transport.ErrRepositoryNotFound) {
+		message.Debugf("Repo not yet available offline, skipping fetch")
+	} else if err != nil {
+		return fmt.Errorf("unable to fetch remote cleanly prior to push: %w", err)
+	}
+
+	// Push all heads and tags to the offline remote
 	err = repo.Push(&git.PushOptions{
 		RemoteName: offlineRemoteName,
 		Auth:       &gitCred,
@@ -108,7 +129,7 @@ func push(localPath, tunnelUrl string, spinner *message.Spinner) error {
 		},
 	})
 
-	if err == git.NoErrAlreadyUpToDate {
+	if errors.Is(err, git.NoErrAlreadyUpToDate) {
 		spinner.Debugf("Repo already up-to-date")
 	} else if err != nil {
 		return fmt.Errorf("unable to push repo to the gitops service: %w", err)

src/internal/packager/deploy.go

@@ -291,7 +291,7 @@ func deployComponents(tempPath tempPaths, component types.ZarfComponent) []types
 		for retry := 0; retry < 3; retry++ {
 			// Push all the repos from the extracted archive
 			if err := git.PushAllDirectories(componentPath.repos); err != nil {
-				message.Errorf(err, "Unable to push repos to the Zarf Registry, retrying in 5 seconds...")
+				message.Errorf(err, "Unable to push repos to the Zarf Repository, retrying in 5 seconds...")
 				time.Sleep(5 * time.Second)
 				continue
 			} else {

src/test/e2e/22_git_and_flux_test.go

@@ -32,6 +32,7 @@ func TestGitAndFlux(t *testing.T) {
 
 	testGitServerConnect(t, tunnel.HttpEndpoint())
 	testGitServerReadOnly(t, tunnel.HttpEndpoint())
+	testGitServerTagAndHash(t, tunnel.HttpEndpoint())
 	waitFluxPodInfoDeployment(t)
 
 	stdOut, stdErr, err = e2e.execZarfCommand("package", "remove", "flux-test", "--confirm")
@@ -41,21 +42,21 @@ func TestGitAndFlux(t *testing.T) {
 	require.NoError(t, err, stdOut, stdErr)
 }
 
-func testGitServerConnect(t *testing.T, gitUrl string) {
+func testGitServerConnect(t *testing.T, gitURL string) {
 	// Make sure Gitea comes up cleanly
-	resp, err := http.Get(gitUrl + "/explore/repos")
+	resp, err := http.Get(gitURL + "/explore/repos")
 	assert.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 }
 
-func testGitServerReadOnly(t *testing.T, gitUrl string) {
+func testGitServerReadOnly(t *testing.T, gitURL string) {
 	// Init the state variable
 	state := k8s.LoadZarfState()
 	config.InitState(state)
 
 	// Get the repo as the readonly user
 	repoName := "mirror__repo1.dso.mil__platform-one__big-bang__apps__security-tools__twistlock"
-	getRepoRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s", gitUrl, config.ZarfGitPushUser, repoName), nil)
+	getRepoRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s", gitURL, config.ZarfGitPushUser, repoName), nil)
 	getRepoResponseBody, err := git.DoHttpThings(getRepoRequest, config.ZarfGitReadUser, config.GetSecret(config.StateGitPull))
 	assert.NoError(t, err)
 
@@ -68,6 +69,36 @@ func testGitServerReadOnly(t *testing.T, gitUrl string) {
 	assert.True(t, permissionsMap["pull"].(bool))
 }
 
+func testGitServerTagAndHash(t *testing.T, gitURL string) {
+	// Init the state variable
+	state := k8s.LoadZarfState()
+	config.InitState(state)
+
+	repoName := "mirror__github.com__defenseunicorns__zarf"
+
+	// Get the Zarf repo tag
+	repoTag := "v0.15.0"
+	getRepoTagsRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s/tags/%s", gitURL, config.ZarfGitPushUser, repoName, repoTag), nil)
+	getRepoTagsResponseBody, err := git.DoHttpThings(getRepoTagsRequest, config.ZarfGitReadUser, config.GetSecret(config.StateGitPull))
+	assert.NoError(t, err)
+
+	// Make sure the pushed tag exists
+	var tagMap map[string]interface{}
+	json.Unmarshal(getRepoTagsResponseBody, &tagMap)
+	assert.Equal(t, repoTag, tagMap["name"])
+
+	// Get the Zarf repo commit
+	repoHash := "c74e2e9626da0400e0a41e78319b3054c53a5d4e"
+	getRepoCommitsRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s/commits", gitURL, config.ZarfGitPushUser, repoName), nil)
+	getRepoCommitsResponseBody, err := git.DoHttpThings(getRepoCommitsRequest, config.ZarfGitReadUser, config.GetSecret(config.StateGitPull))
+	assert.NoError(t, err)
+
+	// Make sure the pushed commit exists
+	var commitMap []map[string]interface{}
+	json.Unmarshal(getRepoCommitsResponseBody, &commitMap)
+	assert.Equal(t, repoHash, commitMap[0]["sha"])
+}
+
 func waitFluxPodInfoDeployment(t *testing.T) {
 	// Deploy the flux example and verify that it works
 	path := fmt.Sprintf("build/zarf-package-flux-test-%s.tar.zst", e2e.arch)