From 66646a36c9a311ad14f5a3b8aaf8273fcdb61872 Mon Sep 17 00:00:00 2001 From: Gusted Date: Sun, 23 Jan 2022 20:30:39 +0100 Subject: [PATCH 1/8] Fix partial cloning a repo (#18373) - Backport from: #18373 - Backport isn't 1-1, because the frontport had a refactor in that area, which v1.16 doesn't have. --- routers/web/repo/http.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/web/repo/http.go b/routers/web/repo/http.go index 6ebf6a789abc3..f76c20af9b124 100644 --- a/routers/web/repo/http.go +++ b/routers/web/repo/http.go @@ -494,6 +494,7 @@ func serviceRPC(h serviceHandler, service string) { cmd := exec.CommandContext(ctx, git.GitExecutable, service, "--stateless-rpc", h.dir) cmd.Dir = h.dir cmd.Env = append(os.Environ(), h.environ...) + cmd.Env = append(cmd.Env, git.GlobalCommandArgs...) cmd.Stdout = h.w cmd.Stdin = reqBody cmd.Stderr = &stderr From 368b34ce14ca92538a5c2ad8813868a24a03dc03 Mon Sep 17 00:00:00 2001 From: Gusted Date: Sun, 23 Jan 2022 20:35:39 +0100 Subject: [PATCH 2/8] Include diff & use copy --- modules/git/diff.go | 1 + routers/web/repo/http.go | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/git/diff.go b/modules/git/diff.go index f90f911be0725..f5c2f649826d8 100644 --- a/modules/git/diff.go +++ b/modules/git/diff.go @@ -90,6 +90,7 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff cmd.Dir = repo.Path cmd.Stdout = writer cmd.Stderr = stderr + copy(cmd.Args, GlobalCommandArgs) if err = cmd.Run(); err != nil { return fmt.Errorf("Run: %v - %s", err, stderr) diff --git a/routers/web/repo/http.go b/routers/web/repo/http.go index f76c20af9b124..8056934d2c458 100644 --- a/routers/web/repo/http.go +++ b/routers/web/repo/http.go @@ -493,8 +493,12 @@ func serviceRPC(h serviceHandler, service string) { var stderr bytes.Buffer cmd := exec.CommandContext(ctx, git.GitExecutable, service, "--stateless-rpc", h.dir) cmd.Dir = h.dir + cmd.Env = append(os.Environ(), h.environ...) - cmd.Env = append(cmd.Env, git.GlobalCommandArgs...) + cargs := make([]string, len(git.GlobalCommandArgs)) + copy(cargs, git.GlobalCommandArgs) + cmd.Env = append(cmd.Env, cargs...) + cmd.Stdout = h.w cmd.Stdin = reqBody cmd.Stderr = &stderr From fce7ea7f4da9c74303e4dddec47228510305e668 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 23 Jan 2022 22:12:32 +0800 Subject: [PATCH 3/8] Add partial clone test --- integrations/git_helper_for_declarative_test.go | 11 +++++++++++ integrations/git_test.go | 6 ++++++ modules/git/repo.go | 5 ++++- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/integrations/git_helper_for_declarative_test.go b/integrations/git_helper_for_declarative_test.go index 8105bcab8dc52..05a3e877756db 100644 --- a/integrations/git_helper_for_declarative_test.go +++ b/integrations/git_helper_for_declarative_test.go @@ -123,6 +123,17 @@ func doGitClone(dstLocalPath string, u *url.URL) func(*testing.T) { } } +func doPartialGitClone(dstLocalPath string, u *url.URL) func(*testing.T) { + return func(t *testing.T) { + assert.NoError(t, git.CloneWithArgs(context.Background(), u.String(), dstLocalPath, allowLFSFilters(), git.CloneRepoOptions{ + Filter: "blob:none", + })) + exist, err := util.IsExist(filepath.Join(dstLocalPath, "README.md")) + assert.NoError(t, err) + assert.True(t, exist) + } +} + func doGitCloneFail(u *url.URL) func(*testing.T) { return func(t *testing.T) { tmpDir, err := os.MkdirTemp("", "doGitCloneFail") diff --git a/integrations/git_test.go b/integrations/git_test.go index 0d33c786aaefa..f8e3b8dd3f782 100644 --- a/integrations/git_test.go +++ b/integrations/git_test.go @@ -69,6 +69,12 @@ func testGit(t *testing.T, u *url.URL) { t.Run("Clone", doGitClone(dstPath, u)) + dstPath2, err := os.MkdirTemp("", httpContext.Reponame) + assert.NoError(t, err) + defer util.RemoveAll(dstPath2) + + t.Run("Partial Clone", doPartialGitClone(dstPath2, u)) + little, big := standardCommitAndPushTest(t, dstPath) littleLFS, bigLFS := lfsCommitAndPushTest(t, dstPath) rawTest(t, &httpContext, little, big, littleLFS, bigLFS) diff --git a/modules/git/repo.go b/modules/git/repo.go index b19e80cd4490d..47e46de3aa6ae 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -101,6 +101,7 @@ type CloneRepoOptions struct { Shared bool NoCheckout bool Depth int + Filter string } // Clone clones original repository to target path. @@ -141,7 +142,9 @@ func CloneWithArgs(ctx context.Context, from, to string, args []string, opts Clo if opts.Depth > 0 { cmd.AddArguments("--depth", strconv.Itoa(opts.Depth)) } - + if opts.Filter != "" { + cmd.AddArguments("--filter", opts.Filter) + } if len(opts.Branch) > 0 { cmd.AddArguments("-b", opts.Branch) } From 027531c36a28bdb726e457267d5fdc07b923361c Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 23 Jan 2022 21:50:20 +0100 Subject: [PATCH 4/8] patch --- modules/git/diff.go | 2 +- routers/web/repo/http.go | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/git/diff.go b/modules/git/diff.go index f5c2f649826d8..1e7b6a7058d51 100644 --- a/modules/git/diff.go +++ b/modules/git/diff.go @@ -90,7 +90,7 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff cmd.Dir = repo.Path cmd.Stdout = writer cmd.Stderr = stderr - copy(cmd.Args, GlobalCommandArgs) + cmd.Args = append(cmd.Args, GlobalCommandArgs...) if err = cmd.Run(); err != nil { return fmt.Errorf("Run: %v - %s", err, stderr) diff --git a/routers/web/repo/http.go b/routers/web/repo/http.go index 8056934d2c458..ff05ccee5f34c 100644 --- a/routers/web/repo/http.go +++ b/routers/web/repo/http.go @@ -495,9 +495,7 @@ func serviceRPC(h serviceHandler, service string) { cmd.Dir = h.dir cmd.Env = append(os.Environ(), h.environ...) - cargs := make([]string, len(git.GlobalCommandArgs)) - copy(cargs, git.GlobalCommandArgs) - cmd.Env = append(cmd.Env, cargs...) + cmd.Args = append(cmd.Args, git.GlobalCommandArgs...) cmd.Stdout = h.w cmd.Stdin = reqBody From 09c1a2bdab4c094a1b1a444d0b165150ee36d953 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 23 Jan 2022 21:51:18 +0100 Subject: [PATCH 5/8] Apply suggestions from code review --- routers/web/repo/http.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/routers/web/repo/http.go b/routers/web/repo/http.go index ff05ccee5f34c..d552bdfe1a3bf 100644 --- a/routers/web/repo/http.go +++ b/routers/web/repo/http.go @@ -493,10 +493,8 @@ func serviceRPC(h serviceHandler, service string) { var stderr bytes.Buffer cmd := exec.CommandContext(ctx, git.GitExecutable, service, "--stateless-rpc", h.dir) cmd.Dir = h.dir - cmd.Env = append(os.Environ(), h.environ...) cmd.Args = append(cmd.Args, git.GlobalCommandArgs...) - cmd.Stdout = h.w cmd.Stdin = reqBody cmd.Stderr = &stderr From 2acce707c87ca53496448c9fb0236096307bb025 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 23 Jan 2022 21:53:09 +0100 Subject: [PATCH 6/8] globalArgs first --- modules/git/diff.go | 3 ++- routers/web/repo/http.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/git/diff.go b/modules/git/diff.go index 1e7b6a7058d51..9215fbcd8f54d 100644 --- a/modules/git/diff.go +++ b/modules/git/diff.go @@ -90,7 +90,8 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff cmd.Dir = repo.Path cmd.Stdout = writer cmd.Stderr = stderr - cmd.Args = append(cmd.Args, GlobalCommandArgs...) + gArgs := git.GlobalCommandArgs + cmd.Args = append(gArgs, cmd.Args...) if err = cmd.Run(); err != nil { return fmt.Errorf("Run: %v - %s", err, stderr) diff --git a/routers/web/repo/http.go b/routers/web/repo/http.go index d552bdfe1a3bf..6ce32f0e51c16 100644 --- a/routers/web/repo/http.go +++ b/routers/web/repo/http.go @@ -494,7 +494,8 @@ func serviceRPC(h serviceHandler, service string) { cmd := exec.CommandContext(ctx, git.GitExecutable, service, "--stateless-rpc", h.dir) cmd.Dir = h.dir cmd.Env = append(os.Environ(), h.environ...) - cmd.Args = append(cmd.Args, git.GlobalCommandArgs...) + gArgs := git.GlobalCommandArgs + cmd.Args = append(gArgs, cmd.Args...) cmd.Stdout = h.w cmd.Stdin = reqBody cmd.Stderr = &stderr From 1ef5b6173ca1fdb0a7907acbd2af5f4765d12fe6 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 23 Jan 2022 22:09:54 +0100 Subject: [PATCH 7/8] avoid copy but make GlobalCMDArgs append first --- modules/git/diff.go | 17 ++++++++--------- routers/web/repo/http.go | 5 ++--- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/modules/git/diff.go b/modules/git/diff.go index 9215fbcd8f54d..8ca58b4809570 100644 --- a/modules/git/diff.go +++ b/modules/git/diff.go @@ -59,27 +59,28 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff ctx, _, finished := process.GetManager().AddContext(repo.Ctx, fmt.Sprintf("GetRawDiffForFile: [repo_path: %s]", repo.Path)) defer finished() - var cmd *exec.Cmd + cmd := exec.CommandContext(ctx, GitExecutable, GlobalCommandArgs...) + switch diffType { case RawDiffNormal: if len(startCommit) != 0 { - cmd = exec.CommandContext(ctx, GitExecutable, append([]string{"diff", "-M", startCommit, endCommit}, fileArgs...)...) + cmd.Args = append(cmd.Args, append([]string{"diff", "-M", startCommit, endCommit}, fileArgs...)...) } else if commit.ParentCount() == 0 { - cmd = exec.CommandContext(ctx, GitExecutable, append([]string{"show", endCommit}, fileArgs...)...) + cmd.Args = append(cmd.Args, append([]string{"show", endCommit}, fileArgs...)...) } else { c, _ := commit.Parent(0) - cmd = exec.CommandContext(ctx, GitExecutable, append([]string{"diff", "-M", c.ID.String(), endCommit}, fileArgs...)...) + cmd.Args = append(cmd.Args, append([]string{"diff", "-M", c.ID.String(), endCommit}, fileArgs...)...) } case RawDiffPatch: if len(startCommit) != 0 { query := fmt.Sprintf("%s...%s", endCommit, startCommit) - cmd = exec.CommandContext(ctx, GitExecutable, append([]string{"format-patch", "--no-signature", "--stdout", "--root", query}, fileArgs...)...) + cmd.Args = append(cmd.Args, append([]string{"format-patch", "--no-signature", "--stdout", "--root", query}, fileArgs...)...) } else if commit.ParentCount() == 0 { - cmd = exec.CommandContext(ctx, GitExecutable, append([]string{"format-patch", "--no-signature", "--stdout", "--root", endCommit}, fileArgs...)...) + cmd.Args = append(cmd.Args, append([]string{"format-patch", "--no-signature", "--stdout", "--root", endCommit}, fileArgs...)...) } else { c, _ := commit.Parent(0) query := fmt.Sprintf("%s...%s", endCommit, c.ID.String()) - cmd = exec.CommandContext(ctx, GitExecutable, append([]string{"format-patch", "--no-signature", "--stdout", query}, fileArgs...)...) + cmd.Args = append(cmd.Args, append([]string{"format-patch", "--no-signature", "--stdout", query}, fileArgs...)...) } default: return fmt.Errorf("invalid diffType: %s", diffType) @@ -90,8 +91,6 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff cmd.Dir = repo.Path cmd.Stdout = writer cmd.Stderr = stderr - gArgs := git.GlobalCommandArgs - cmd.Args = append(gArgs, cmd.Args...) if err = cmd.Run(); err != nil { return fmt.Errorf("Run: %v - %s", err, stderr) diff --git a/routers/web/repo/http.go b/routers/web/repo/http.go index 6ce32f0e51c16..d21468dba55cc 100644 --- a/routers/web/repo/http.go +++ b/routers/web/repo/http.go @@ -491,11 +491,10 @@ func serviceRPC(h serviceHandler, service string) { defer finished() var stderr bytes.Buffer - cmd := exec.CommandContext(ctx, git.GitExecutable, service, "--stateless-rpc", h.dir) + args := append(git.GlobalCommandArgs, []string{service, "--stateless-rpc", h.dir}...) + cmd := exec.CommandContext(ctx, git.GitExecutable, args...) cmd.Dir = h.dir cmd.Env = append(os.Environ(), h.environ...) - gArgs := git.GlobalCommandArgs - cmd.Args = append(gArgs, cmd.Args...) cmd.Stdout = h.w cmd.Stdin = reqBody cmd.Stderr = &stderr From 3a6e2429b4b0be505b6af077d2c86577bc795727 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 23 Jan 2022 22:17:06 +0100 Subject: [PATCH 8/8] please linter --- routers/web/repo/http.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/routers/web/repo/http.go b/routers/web/repo/http.go index d21468dba55cc..f5ca7f1856d0c 100644 --- a/routers/web/repo/http.go +++ b/routers/web/repo/http.go @@ -491,7 +491,9 @@ func serviceRPC(h serviceHandler, service string) { defer finished() var stderr bytes.Buffer - args := append(git.GlobalCommandArgs, []string{service, "--stateless-rpc", h.dir}...) + args := make([]string, len(git.GlobalCommandArgs)) + copy(args, git.GlobalCommandArgs) + args = append(args, []string{service, "--stateless-rpc", h.dir}...) cmd := exec.CommandContext(ctx, git.GitExecutable, args...) cmd.Dir = h.dir cmd.Env = append(os.Environ(), h.environ...)