From 55d80fa82f3fb9843b2dfcf27bf31d39349dea1b Mon Sep 17 00:00:00 2001 From: Eran Turgeman <81029514+eranturgeman@users.noreply.github.com> Date: Sun, 20 Oct 2024 12:59:19 +0300 Subject: [PATCH] Enable using local repository instead cloning a remote repo - ScanRepository (#767) --- azure_test.go | 17 +++++--- bitbucket_server_test.go | 17 +++++--- github_test.go | 17 +++++--- gitlab_test.go | 17 +++++--- integrationutils.go | 66 ++++++++++++++++++++++---------- scanrepository/scanrepository.go | 38 +++++++++++++----- utils/consts.go | 11 +++--- utils/git.go | 12 ++++-- utils/params.go | 6 +++ 9 files changed, 139 insertions(+), 62 deletions(-) diff --git a/azure_test.go b/azure_test.go index 818267f48..f4e3101ba 100644 --- a/azure_test.go +++ b/azure_test.go @@ -20,25 +20,30 @@ func buildAzureReposClient(t *testing.T, azureToken string) vcsclient.VcsClient return azureClient } -func buildAzureReposIntegrationTestDetails(t *testing.T) *IntegrationTestDetails { +func buildAzureReposIntegrationTestDetails(t *testing.T, useLocalRepo bool) *IntegrationTestDetails { integrationRepoToken := getIntegrationToken(t, azureIntegrationTokenEnv) - testDetails := NewIntegrationTestDetails(integrationRepoToken, string(utils.AzureRepos), azureGitCloneUrl, "frogbot-test") + testDetails := NewIntegrationTestDetails(integrationRepoToken, string(utils.AzureRepos), azureGitCloneUrl, "frogbot-test", useLocalRepo) testDetails.ApiEndpoint = azureApiEndpoint return testDetails } -func azureReposTestsInit(t *testing.T) (vcsclient.VcsClient, *IntegrationTestDetails) { - testDetails := buildAzureReposIntegrationTestDetails(t) +func azureReposTestsInit(t *testing.T, useLocalRepo bool) (vcsclient.VcsClient, *IntegrationTestDetails) { + testDetails := buildAzureReposIntegrationTestDetails(t, useLocalRepo) azureClient := buildAzureReposClient(t, testDetails.GitToken) return azureClient, testDetails } func TestAzureRepos_ScanPullRequestIntegration(t *testing.T) { - azureClient, testDetails := azureReposTestsInit(t) + azureClient, testDetails := azureReposTestsInit(t, false) runScanPullRequestCmd(t, azureClient, testDetails) } func TestAzureRepos_ScanRepositoryIntegration(t *testing.T) { - azureClient, testDetails := azureReposTestsInit(t) + azureClient, testDetails := azureReposTestsInit(t, false) + runScanRepositoryCmd(t, azureClient, testDetails) +} + +func TestAzureRepos_ScanRepositoryWithLocalDirIntegration(t *testing.T) { + azureClient, testDetails := azureReposTestsInit(t, true) runScanRepositoryCmd(t, azureClient, testDetails) } diff --git a/bitbucket_server_test.go b/bitbucket_server_test.go index a119ebc31..fe3531c3f 100644 --- a/bitbucket_server_test.go +++ b/bitbucket_server_test.go @@ -26,9 +26,9 @@ func buildBitbucketServerClient(t *testing.T, bitbucketServerToken string) vcscl return bbClient } -func buildBitbucketServerIntegrationTestDetails(t *testing.T) *IntegrationTestDetails { +func buildBitbucketServerIntegrationTestDetails(t *testing.T, useLocalRepo bool) *IntegrationTestDetails { integrationRepoToken := getIntegrationToken(t, bitbucketServerIntegrationTokenEnv) - testDetails := NewIntegrationTestDetails(integrationRepoToken, string(utils.BitbucketServer), bitbucketServerGitCloneUrl, "FROG") + testDetails := NewIntegrationTestDetails(integrationRepoToken, string(utils.BitbucketServer), bitbucketServerGitCloneUrl, "FROG", useLocalRepo) testDetails.ApiEndpoint = bitbucketServerApiEndpoint return testDetails } @@ -53,19 +53,24 @@ func waitForConnection(t *testing.T) { require.NoError(t, retryExecutor.Execute()) } -func bitbucketServerTestsInit(t *testing.T) (vcsclient.VcsClient, *IntegrationTestDetails) { - testDetails := buildBitbucketServerIntegrationTestDetails(t) +func bitbucketServerTestsInit(t *testing.T, useLocalRepo bool) (vcsclient.VcsClient, *IntegrationTestDetails) { + testDetails := buildBitbucketServerIntegrationTestDetails(t, useLocalRepo) bbClient := buildBitbucketServerClient(t, testDetails.GitToken) waitForConnection(t) return bbClient, testDetails } func TestBitbucketServer_ScanPullRequestIntegration(t *testing.T) { - bbClient, testDetails := bitbucketServerTestsInit(t) + bbClient, testDetails := bitbucketServerTestsInit(t, false) runScanPullRequestCmd(t, bbClient, testDetails) } func TestBitbucketServer_ScanRepositoryIntegration(t *testing.T) { - bbClient, testDetails := bitbucketServerTestsInit(t) + bbClient, testDetails := bitbucketServerTestsInit(t, false) + runScanRepositoryCmd(t, bbClient, testDetails) +} + +func TestBitbucketServer_ScanRepositoryWithLocalDirIntegration(t *testing.T) { + bbClient, testDetails := bitbucketServerTestsInit(t, true) runScanRepositoryCmd(t, bbClient, testDetails) } diff --git a/github_test.go b/github_test.go index 46fd658f2..cbdf5c723 100644 --- a/github_test.go +++ b/github_test.go @@ -19,23 +19,28 @@ func buildGitHubClient(t *testing.T, githubToken string) vcsclient.VcsClient { return githubClient } -func buildGitHubIntegrationTestDetails(t *testing.T) *IntegrationTestDetails { +func buildGitHubIntegrationTestDetails(t *testing.T, useLocalRepo bool) *IntegrationTestDetails { integrationRepoToken := getIntegrationToken(t, githubIntegrationTokenEnv) - return NewIntegrationTestDetails(integrationRepoToken, string(utils.GitHub), githubGitCloneUrl, "frogbot-test") + return NewIntegrationTestDetails(integrationRepoToken, string(utils.GitHub), githubGitCloneUrl, "frogbot-test", useLocalRepo) } -func githubTestsInit(t *testing.T) (vcsclient.VcsClient, *IntegrationTestDetails) { - testDetails := buildGitHubIntegrationTestDetails(t) +func githubTestsInit(t *testing.T, useLocalRepo bool) (vcsclient.VcsClient, *IntegrationTestDetails) { + testDetails := buildGitHubIntegrationTestDetails(t, useLocalRepo) githubClient := buildGitHubClient(t, testDetails.GitToken) return githubClient, testDetails } func TestGitHub_ScanPullRequestIntegration(t *testing.T) { - githubClient, testDetails := githubTestsInit(t) + githubClient, testDetails := githubTestsInit(t, false) runScanPullRequestCmd(t, githubClient, testDetails) } func TestGitHub_ScanRepositoryIntegration(t *testing.T) { - githubClient, testDetails := githubTestsInit(t) + githubClient, testDetails := githubTestsInit(t, false) + runScanRepositoryCmd(t, githubClient, testDetails) +} + +func TestGitHub_ScanRepositoryWithLocalDirIntegration(t *testing.T) { + githubClient, testDetails := githubTestsInit(t, true) runScanRepositoryCmd(t, githubClient, testDetails) } diff --git a/gitlab_test.go b/gitlab_test.go index 243d51817..b7ed01a90 100644 --- a/gitlab_test.go +++ b/gitlab_test.go @@ -19,23 +19,28 @@ func buildGitLabClient(t *testing.T, gitlabToken string) vcsclient.VcsClient { return azureClient } -func buildGitLabIntegrationTestDetails(t *testing.T) *IntegrationTestDetails { +func buildGitLabIntegrationTestDetails(t *testing.T, useLocalRepo bool) *IntegrationTestDetails { integrationRepoToken := getIntegrationToken(t, gitlabIntegrationTokenEnv) - return NewIntegrationTestDetails(integrationRepoToken, string(utils.GitLab), gitlabGitCloneUrl, "frogbot-test2") + return NewIntegrationTestDetails(integrationRepoToken, string(utils.GitLab), gitlabGitCloneUrl, "frogbot-test2", useLocalRepo) } -func gitlabTestsInit(t *testing.T) (vcsclient.VcsClient, *IntegrationTestDetails) { - testDetails := buildGitLabIntegrationTestDetails(t) +func gitlabTestsInit(t *testing.T, useLocalRepo bool) (vcsclient.VcsClient, *IntegrationTestDetails) { + testDetails := buildGitLabIntegrationTestDetails(t, useLocalRepo) gitlabClient := buildGitLabClient(t, testDetails.GitToken) return gitlabClient, testDetails } func TestGitLab_ScanPullRequestIntegration(t *testing.T) { - gitlabClient, testDetails := gitlabTestsInit(t) + gitlabClient, testDetails := gitlabTestsInit(t, false) runScanPullRequestCmd(t, gitlabClient, testDetails) } func TestGitLab_ScanRepositoryIntegration(t *testing.T) { - gitlabClient, testDetails := gitlabTestsInit(t) + gitlabClient, testDetails := gitlabTestsInit(t, false) + runScanRepositoryCmd(t, gitlabClient, testDetails) +} + +func TestGitLab_ScanRepositoryWithLocalDirIntegration(t *testing.T) { + gitlabClient, testDetails := gitlabTestsInit(t, true) runScanRepositoryCmd(t, gitlabClient, testDetails) } diff --git a/integrationutils.go b/integrationutils.go index c8538f0e5..b749faa90 100644 --- a/integrationutils.go +++ b/integrationutils.go @@ -3,6 +3,8 @@ package main import ( "context" "fmt" + "github.com/go-git/go-git/v5" + githttp "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/jfrog/frogbot/v2/scanpullrequest" "github.com/jfrog/frogbot/v2/scanrepository" "github.com/jfrog/frogbot/v2/utils" @@ -36,17 +38,19 @@ type IntegrationTestDetails struct { ApiEndpoint string PullRequestID string CustomBranchName string + UseLocalRepo bool } -func NewIntegrationTestDetails(token, gitProvider, gitCloneUrl, repoOwner string) *IntegrationTestDetails { +func NewIntegrationTestDetails(token, gitProvider, gitCloneUrl, repoOwner string, useLocalRepo bool) *IntegrationTestDetails { return &IntegrationTestDetails{ - GitProject: repoName, - RepoOwner: repoOwner, - RepoName: repoName, - GitToken: token, - GitUsername: "frogbot", - GitProvider: gitProvider, - GitCloneURL: gitCloneUrl, + GitProject: repoName, + RepoOwner: repoOwner, + RepoName: repoName, + GitToken: token, + GitUsername: "frogbot", + GitProvider: gitProvider, + GitCloneURL: gitCloneUrl, + UseLocalRepo: useLocalRepo, } } @@ -71,18 +75,23 @@ func setIntegrationTestEnvs(t *testing.T, testDetails *IntegrationTestDetails) f // Frogbot sanitizes all the environment variables that start with 'JF', // so we restore them at the end of the test to avoid collisions with other tests envRestoreFunc := getJfrogEnvRestoreFunc(t) + useLocalRepo := "false" + if testDetails.UseLocalRepo { + useLocalRepo = "true" + } unsetEnvs := utils.SetEnvsAndAssertWithCallback(t, map[string]string{ - utils.RequirementsFileEnv: "requirements.txt", - utils.GitPullRequestIDEnv: testDetails.PullRequestID, - utils.GitProvider: testDetails.GitProvider, - utils.GitTokenEnv: testDetails.GitToken, - utils.GitRepoEnv: testDetails.RepoName, - utils.GitRepoOwnerEnv: testDetails.RepoOwner, - utils.BranchNameTemplateEnv: testDetails.CustomBranchName, - utils.GitApiEndpointEnv: testDetails.ApiEndpoint, - utils.GitProjectEnv: testDetails.GitProject, - utils.GitUsernameEnv: testDetails.GitUsername, - utils.GitBaseBranchEnv: mainBranch, + utils.RequirementsFileEnv: "requirements.txt", + utils.GitPullRequestIDEnv: testDetails.PullRequestID, + utils.GitProvider: testDetails.GitProvider, + utils.GitTokenEnv: testDetails.GitToken, + utils.GitRepoEnv: testDetails.RepoName, + utils.GitRepoOwnerEnv: testDetails.RepoOwner, + utils.BranchNameTemplateEnv: testDetails.CustomBranchName, + utils.GitApiEndpointEnv: testDetails.ApiEndpoint, + utils.GitProjectEnv: testDetails.GitProject, + utils.GitUsernameEnv: testDetails.GitUsername, + utils.GitBaseBranchEnv: mainBranch, + utils.GitUseLocalRepositoryEnv: useLocalRepo, }) return func() { envRestoreFunc() @@ -173,11 +182,28 @@ func runScanPullRequestCmd(t *testing.T, client vcsclient.VcsClient, testDetails } func runScanRepositoryCmd(t *testing.T, client vcsclient.VcsClient, testDetails *IntegrationTestDetails) { - _, restoreFunc := utils.ChangeToTempDirWithCallback(t) + testTempDir, restoreFunc := utils.ChangeToTempDirWithCallback(t) defer func() { assert.NoError(t, restoreFunc()) }() + // When testing using local repository clone the repository before the test starts so we can work with it as if it existed locally + if testDetails.UseLocalRepo { + cloneOptions := &git.CloneOptions{ + URL: testDetails.GitCloneURL, + Auth: &githttp.BasicAuth{ + Username: testDetails.GitUsername, + Password: testDetails.GitToken, + }, + RemoteName: "origin", + ReferenceName: utils.GetFullBranchName("main"), + SingleBranch: true, + Depth: 1, + Tags: git.NoTags, + } + _, err := git.PlainClone(testTempDir, false, cloneOptions) + require.NoError(t, err) + } timestamp := getTimestamp() // Add a timestamp to the fixing pull requests, to identify them later testDetails.CustomBranchName = "frogbot-{IMPACTED_PACKAGE}-{BRANCH_NAME_HASH}-" + timestamp diff --git a/scanrepository/scanrepository.go b/scanrepository/scanrepository.go index 7ce45018a..17742b8ab 100644 --- a/scanrepository/scanrepository.go +++ b/scanrepository/scanrepository.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + biutils "github.com/jfrog/build-info-go/utils" + "os" "path/filepath" "regexp" "strings" @@ -80,17 +82,17 @@ func (cfp *ScanRepositoryCmd) scanAndFixBranch(repository *utils.Repository) (er cfp.analyticsService.UpdateAndSendXscAnalyticsGeneralEventFinalize(err) }() - clonedRepoDir, restoreBaseDir, err := cfp.cloneRepositoryAndCheckoutToBranch() + repoDir, restoreBaseDir, err := cfp.cloneRepositoryOrUseLocalAndCheckoutToBranch() if err != nil { return } - cfp.baseWd = clonedRepoDir + cfp.baseWd = repoDir defer func() { // On dry run don't delete the folder as we want to validate results if cfp.dryRun { return } - err = errors.Join(err, restoreBaseDir(), fileutils.RemoveTempDir(clonedRepoDir)) + err = errors.Join(err, restoreBaseDir(), fileutils.RemoveTempDir(repoDir)) }() // If MSI exists we always need to report events @@ -476,7 +478,7 @@ func (cfp *ScanRepositoryCmd) preparePullRequestDetails(vulnerabilitiesDetails . return pullRequestTitle, prBody, extraComments, nil } -func (cfp *ScanRepositoryCmd) cloneRepositoryAndCheckoutToBranch() (tempWd string, restoreDir func() error, err error) { +func (cfp *ScanRepositoryCmd) cloneRepositoryOrUseLocalAndCheckoutToBranch() (tempWd string, restoreDir func() error, err error) { if cfp.dryRun { tempWd = filepath.Join(cfp.dryRunRepoPath, cfp.scanDetails.RepoName) } else { @@ -487,13 +489,29 @@ func (cfp *ScanRepositoryCmd) cloneRepositoryAndCheckoutToBranch() (tempWd strin } log.Debug("Created temp working directory:", tempWd) - // Clone the content of the repo to the new working directory - if err = cfp.gitManager.Clone(tempWd, cfp.scanDetails.BaseBranch()); err != nil { - return + if cfp.scanDetails.UseLocalRepository { + var curDir string + if curDir, err = os.Getwd(); err != nil { + return + } + if err = biutils.CopyDir(curDir, tempWd, true, nil); err != nil { + return + } + // 'CD' into the temp working directory + restoreDir, err = utils.Chdir(tempWd) + if err != nil { + return + } + // Set the current copied local dir as the local git repository we are working with + err = cfp.gitManager.SetLocalRepository() + } else { + // Clone the content of the repo to the new working directory + if err = cfp.gitManager.Clone(tempWd, cfp.scanDetails.BaseBranch()); err != nil { + return + } + // 'CD' into the temp working directory + restoreDir, err = utils.Chdir(tempWd) } - - // 'CD' into the temp working directory - restoreDir, err = utils.Chdir(tempWd) return } diff --git a/utils/consts.go b/utils/consts.go index 7fbf5d953..5730d8c00 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -34,11 +34,12 @@ const ( JfrogConfigProfileEnv = "JF_CONFIG_PROFILE" // Git environment variables - GitProvider = "JF_GIT_PROVIDER" - GitRepoOwnerEnv = "JF_GIT_OWNER" - GitRepoEnv = "JF_GIT_REPO" - GitProjectEnv = "JF_GIT_PROJECT" - GitUsernameEnv = "JF_GIT_USERNAME" + GitProvider = "JF_GIT_PROVIDER" + GitRepoOwnerEnv = "JF_GIT_OWNER" + GitRepoEnv = "JF_GIT_REPO" + GitProjectEnv = "JF_GIT_PROJECT" + GitUsernameEnv = "JF_GIT_USERNAME" + GitUseLocalRepositoryEnv = "JF_USE_LOCAL_REPOSITORY" // Git naming template environment variables BranchNameTemplateEnv = "JF_BRANCH_NAME_TEMPLATE" diff --git a/utils/git.go b/utils/git.go index 986a3fa87..b1b7b7300 100644 --- a/utils/git.go +++ b/utils/git.go @@ -92,7 +92,7 @@ func (gm *GitManager) SetRemoteGitUrl(remoteHttpsGitUrl string) (*GitManager, er } if gm.localGitRepository == nil { - if _, err = gm.SetLocalRepository(); err != nil { + if _, err = gm.SetLocalRepositoryAndRemoteName(); err != nil { return gm, err } } @@ -115,14 +115,20 @@ func (gm *GitManager) SetRemoteGitUrl(remoteHttpsGitUrl string) (*GitManager, er return gm, nil } -func (gm *GitManager) SetLocalRepository() (*GitManager, error) { +func (gm *GitManager) SetLocalRepositoryAndRemoteName() (*GitManager, error) { var err error // Re-initialize the repository and update remoteName gm.remoteName = vcsutils.RemoteName - gm.localGitRepository, err = git.PlainOpen(".") + err = gm.SetLocalRepository() return gm, err } +func (gm *GitManager) SetLocalRepository() error { + var err error + gm.localGitRepository, err = git.PlainOpen(".") + return err +} + func (gm *GitManager) SetGitParams(gitParams *Git) (*GitManager, error) { var err error if gm.customTemplates, err = loadCustomTemplates(gitParams.CommitMessageTemplate, gitParams.BranchNameTemplate, gitParams.PullRequestTitleTemplate); err != nil { diff --git a/utils/params.go b/utils/params.go index b83216561..6c534b4a6 100644 --- a/utils/params.go +++ b/utils/params.go @@ -302,6 +302,7 @@ type Git struct { AggregateFixes bool `yaml:"aggregateFixes,omitempty"` PullRequestDetails vcsclient.PullRequestInfo RepositoryCloneUrl string + UseLocalRepository bool } func (g *Git) setDefaultsIfNeeded(gitParamsFromEnv *Git, commandName string) (err error) { @@ -373,6 +374,11 @@ func (g *Git) extractScanRepositoryEnvParams(gitParamsFromEnv *Git) (err error) return } } + if !g.UseLocalRepository { + if g.UseLocalRepository, err = getBoolEnv(GitUseLocalRepositoryEnv, false); err != nil { + return + } + } return }