From 98049d74cdc7b2008a379d294c4e28332ee57276 Mon Sep 17 00:00:00 2001 From: ghe Date: Mon, 28 Nov 2022 16:03:05 +0000 Subject: [PATCH] feat: shallow clone a repo with simple-git Support for Github.com to shallow clone a repo with a API token. Clones into a temp dir and deletes it if clone did not succeed. --- package.json | 1 + src/lib/git-clone.ts | 61 +++++++++++++++++ src/lib/index.ts | 1 + .../source-handlers/github/git-clone-url.ts | 8 +++ src/lib/source-handlers/github/index.ts | 1 + test/lib/git-clone.spec.ts | 65 +++++++++++++++++++ .../lib/source-handlers/github/github.test.ts | 23 +++++++ 7 files changed, 160 insertions(+) create mode 100644 src/lib/git-clone.ts create mode 100644 src/lib/source-handlers/github/git-clone-url.ts create mode 100644 test/lib/git-clone.spec.ts diff --git a/package.json b/package.json index 70fc3da2..ff9132b1 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,7 @@ "needle": "2.9.1", "p-map": "4.0.0", "parse-link-header": "2.0.0", + "simple-git": "3.15.0", "sleep-promise": "8.0.1", "snyk-request-manager": "1.8.0", "source-map-support": "^0.5.16", diff --git a/src/lib/git-clone.ts b/src/lib/git-clone.ts new file mode 100644 index 00000000..01ff1992 --- /dev/null +++ b/src/lib/git-clone.ts @@ -0,0 +1,61 @@ +import * as debugLib from 'debug'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; + +import type { SimpleGitOptions } from 'simple-git'; +import { simpleGit } from 'simple-git'; +import * as github from '../lib/source-handlers/github'; +import type { RepoMetaData } from './types'; +import { SupportedIntegrationTypesUpdateProject } from './types'; + +const debug = debugLib('snyk:git-clone'); + +const urlGenerators = { + [SupportedIntegrationTypesUpdateProject.GITHUB]: github.buildGitCloneUrl, +}; + +interface GitCloneResponse { + success: boolean; + repoPath?: string; + gitResponse: string; +} +export async function gitClone( + integrationType: SupportedIntegrationTypesUpdateProject.GITHUB, + meta: RepoMetaData, +): Promise { + const repoClonePath = await fs.mkdtempSync( + path.join(os.tmpdir(), `snyk-clone-${Date.now()}-${Math.random()}`), + ); + try { + const cloneUrl = urlGenerators[integrationType](meta); + const options: Partial = { + baseDir: repoClonePath, + binary: 'git', + maxConcurrentProcesses: 6, + trimmed: false, + }; + debug(`Trying to shallow clone repo: ${meta.cloneUrl}`); + const git = simpleGit(options); + const output = await git.clone(cloneUrl, repoClonePath, { + '--depth': '1', + '--branch': meta.branch, + }); + + debug(`Repo ${meta.cloneUrl} was cloned`); + return { + gitResponse: output, + success: true, + repoPath: repoClonePath, + }; + } catch (err: any) { + debug(`Could not shallow clone the repo:\n ${err}`); + if (fs.existsSync(repoClonePath)) { + fs.rmdirSync(repoClonePath); + } + return { + success: false, + gitResponse: err.message, + }; + } +} diff --git a/src/lib/index.ts b/src/lib/index.ts index d9a04423..bc41c9b5 100644 --- a/src/lib/index.ts +++ b/src/lib/index.ts @@ -12,6 +12,7 @@ export * from './get-snyk-host'; export * from './filter-out-existing-orgs'; export * from './supported-project-types'; export * from './find-files'; +export * from './git-clone'; export * from './source-handlers/github'; export * from './source-handlers/gitlab'; diff --git a/src/lib/source-handlers/github/git-clone-url.ts b/src/lib/source-handlers/github/git-clone-url.ts new file mode 100644 index 00000000..1dc785c0 --- /dev/null +++ b/src/lib/source-handlers/github/git-clone-url.ts @@ -0,0 +1,8 @@ +import type { RepoMetaData } from '../../types'; +import { getGithubToken } from './get-github-token'; + +export function buildGitCloneUrl(meta: RepoMetaData): string { + const { cloneUrl } = meta; + const url = new URL(cloneUrl); + return `${url.protocol}//${getGithubToken()}@${url.hostname}${url.pathname}`; +} diff --git a/src/lib/source-handlers/github/index.ts b/src/lib/source-handlers/github/index.ts index c037bac5..09f12a54 100644 --- a/src/lib/source-handlers/github/index.ts +++ b/src/lib/source-handlers/github/index.ts @@ -4,3 +4,4 @@ export * from './organization-is-empty'; export * from './get-repo-metadata'; export * from './types'; export * from './is-configured'; +export * from './git-clone-url'; diff --git a/test/lib/git-clone.spec.ts b/test/lib/git-clone.spec.ts new file mode 100644 index 00000000..a36a729a --- /dev/null +++ b/test/lib/git-clone.spec.ts @@ -0,0 +1,65 @@ +import * as fs from 'fs'; +import { gitClone } from '../../src/lib'; +import { SupportedIntegrationTypesUpdateProject } from '../../src/lib/types'; + +describe('gitClone', () => { + const OLD_ENV = process.env; + const removeFolders: string[] = []; + afterAll(() => { + process.env = { ...OLD_ENV }; + }); + + afterEach(() => { + for (const f of removeFolders) { + try { + fs.rmdirSync(f, { recursive: true }); + } catch (e) { + console.log('Failed to clean up test', e); + } + } + }); + describe('Github', () => { + it('successfully clones a repo', async () => { + process.env.GITHUB_TOKEN = process.env.GH_TOKEN; + process.env.SNYK_LOG_PATH = __dirname; + + const res = await gitClone( + SupportedIntegrationTypesUpdateProject.GITHUB, + { + branch: 'master', + cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git', + sshUrl: 'git@github.com:snyk-fixtures/monorepo-simple.git', + }, + ); + + expect(res).toEqual({ + gitResponse: '', + repoPath: expect.any(String), + success: true, + }); + removeFolders.push(res.repoPath!); + }, 70000); + + it('fails to clone a repo for non-existent branch', async () => { + process.env.GITHUB_TOKEN = process.env.GH_TOKEN; + process.env.SNYK_LOG_PATH = __dirname; + + const res = await gitClone( + SupportedIntegrationTypesUpdateProject.GITHUB, + { + branch: 'non-existent', + cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git', + sshUrl: 'git@github.com:snyk-fixtures/monorepo-simple.git', + }, + ); + + expect(res).toEqual({ + gitResponse: expect.stringContaining( + 'Remote branch non-existent not found in upstream origin', + ), + success: false, + }); + removeFolders.push(res.repoPath!); + }, 70000); + }); +}); diff --git a/test/lib/source-handlers/github/github.test.ts b/test/lib/source-handlers/github/github.test.ts index 515249b4..06ddbcda 100644 --- a/test/lib/source-handlers/github/github.test.ts +++ b/test/lib/source-handlers/github/github.test.ts @@ -80,3 +80,26 @@ describe('isGithubConfigured', () => { expect(() => github.isGithubConfigured()).toThrow(); }); }); + +describe('buildGitCloneUrl', () => { + const OLD_ENV = process.env; + + beforeEach(async () => { + delete process.env.GITHUB_TOKEN; + }); + + afterEach(async () => { + process.env = { ...OLD_ENV }; + }); + it('builds correct clone url for github.com / ghe (the urls come back from API already correct)', async () => { + process.env.GITHUB_TOKEN = 'secret_token'; + const url = github.buildGitCloneUrl({ + branch: 'main', + sshUrl: 'https://git@github.com:snyk-tech-services/snyk-api-import.git', + cloneUrl: 'https://github.com/snyk-tech-services/snyk-api-import.git', + }); + expect(url).toEqual( + `https://secret_token@github.com/snyk-tech-services/snyk-api-import.git`, + ); + }); +});