Skip to content

Commit

Permalink
feat: shallow clone a repo with simple-git
Browse files Browse the repository at this point in the history
Support for Github.com to shallow clone a repo with
a API token. Clones into a temp dir and deletes it if clone did not succeed.
  • Loading branch information
lili2311 committed Nov 29, 2022
1 parent 85e458e commit 98049d7
Show file tree
Hide file tree
Showing 7 changed files with 160 additions and 0 deletions.
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
"needle": "2.9.1",
"p-map": "4.0.0",
"parse-link-header": "2.0.0",
"simple-git": "3.15.0",
"sleep-promise": "8.0.1",
"snyk-request-manager": "1.8.0",
"source-map-support": "^0.5.16",
Expand Down
61 changes: 61 additions & 0 deletions src/lib/git-clone.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
import * as debugLib from 'debug';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';

import type { SimpleGitOptions } from 'simple-git';
import { simpleGit } from 'simple-git';
import * as github from '../lib/source-handlers/github';
import type { RepoMetaData } from './types';
import { SupportedIntegrationTypesUpdateProject } from './types';

const debug = debugLib('snyk:git-clone');

const urlGenerators = {
[SupportedIntegrationTypesUpdateProject.GITHUB]: github.buildGitCloneUrl,
};

interface GitCloneResponse {
success: boolean;
repoPath?: string;
gitResponse: string;
}
export async function gitClone(
integrationType: SupportedIntegrationTypesUpdateProject.GITHUB,
meta: RepoMetaData,
): Promise<GitCloneResponse> {
const repoClonePath = await fs.mkdtempSync(
path.join(os.tmpdir(), `snyk-clone-${Date.now()}-${Math.random()}`),
);
try {
const cloneUrl = urlGenerators[integrationType](meta);
const options: Partial<SimpleGitOptions> = {
baseDir: repoClonePath,
binary: 'git',
maxConcurrentProcesses: 6,
trimmed: false,
};
debug(`Trying to shallow clone repo: ${meta.cloneUrl}`);
const git = simpleGit(options);
const output = await git.clone(cloneUrl, repoClonePath, {
'--depth': '1',
'--branch': meta.branch,
});

debug(`Repo ${meta.cloneUrl} was cloned`);
return {
gitResponse: output,
success: true,
repoPath: repoClonePath,
};
} catch (err: any) {
debug(`Could not shallow clone the repo:\n ${err}`);
if (fs.existsSync(repoClonePath)) {
fs.rmdirSync(repoClonePath);
}
return {
success: false,
gitResponse: err.message,
};
}
}
1 change: 1 addition & 0 deletions src/lib/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ export * from './get-snyk-host';
export * from './filter-out-existing-orgs';
export * from './supported-project-types';
export * from './find-files';
export * from './git-clone';

export * from './source-handlers/github';
export * from './source-handlers/gitlab';
Expand Down
8 changes: 8 additions & 0 deletions src/lib/source-handlers/github/git-clone-url.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
import type { RepoMetaData } from '../../types';
import { getGithubToken } from './get-github-token';

export function buildGitCloneUrl(meta: RepoMetaData): string {
const { cloneUrl } = meta;
const url = new URL(cloneUrl);
return `${url.protocol}//${getGithubToken()}@${url.hostname}${url.pathname}`;
}
1 change: 1 addition & 0 deletions src/lib/source-handlers/github/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@ export * from './organization-is-empty';
export * from './get-repo-metadata';
export * from './types';
export * from './is-configured';
export * from './git-clone-url';
65 changes: 65 additions & 0 deletions test/lib/git-clone.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
import * as fs from 'fs';
import { gitClone } from '../../src/lib';
import { SupportedIntegrationTypesUpdateProject } from '../../src/lib/types';

describe('gitClone', () => {
const OLD_ENV = process.env;
const removeFolders: string[] = [];
afterAll(() => {
process.env = { ...OLD_ENV };
});

afterEach(() => {
for (const f of removeFolders) {
try {
fs.rmdirSync(f, { recursive: true });
} catch (e) {
console.log('Failed to clean up test', e);
}
}
});
describe('Github', () => {
it('successfully clones a repo', async () => {
process.env.GITHUB_TOKEN = process.env.GH_TOKEN;
process.env.SNYK_LOG_PATH = __dirname;

const res = await gitClone(
SupportedIntegrationTypesUpdateProject.GITHUB,
{
branch: 'master',
cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git',
sshUrl: '[email protected]:snyk-fixtures/monorepo-simple.git',
},
);

expect(res).toEqual({
gitResponse: '',
repoPath: expect.any(String),
success: true,
});
removeFolders.push(res.repoPath!);
}, 70000);

it('fails to clone a repo for non-existent branch', async () => {
process.env.GITHUB_TOKEN = process.env.GH_TOKEN;
process.env.SNYK_LOG_PATH = __dirname;

const res = await gitClone(
SupportedIntegrationTypesUpdateProject.GITHUB,
{
branch: 'non-existent',
cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git',
sshUrl: '[email protected]:snyk-fixtures/monorepo-simple.git',
},
);

expect(res).toEqual({
gitResponse: expect.stringContaining(
'Remote branch non-existent not found in upstream origin',
),
success: false,
});
removeFolders.push(res.repoPath!);
}, 70000);
});
});
23 changes: 23 additions & 0 deletions test/lib/source-handlers/github/github.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,3 +80,26 @@ describe('isGithubConfigured', () => {
expect(() => github.isGithubConfigured()).toThrow();
});
});

describe('buildGitCloneUrl', () => {
const OLD_ENV = process.env;

beforeEach(async () => {
delete process.env.GITHUB_TOKEN;
});

afterEach(async () => {
process.env = { ...OLD_ENV };
});
it('builds correct clone url for github.com / ghe (the urls come back from API already correct)', async () => {
process.env.GITHUB_TOKEN = 'secret_token';
const url = github.buildGitCloneUrl({
branch: 'main',
sshUrl: 'https://[email protected]:snyk-tech-services/snyk-api-import.git',
cloneUrl: 'https://github.com/snyk-tech-services/snyk-api-import.git',
});
expect(url).toEqual(
`https://[email protected]/snyk-tech-services/snyk-api-import.git`,
);
});
});

0 comments on commit 98049d7

Please sign in to comment.