feat: shallow clone a repo with simple-git

package.json

@@ -51,6 +51,7 @@
     "needle": "2.9.1",
     "p-map": "4.0.0",
     "parse-link-header": "2.0.0",
+    "simple-git": "3.15.0",
     "sleep-promise": "8.0.1",
     "snyk-request-manager": "1.8.0",
     "source-map-support": "^0.5.16",

src/lib/git-clone.ts

@@ -0,0 +1,61 @@
+import * as debugLib from 'debug';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+
+import type { SimpleGitOptions } from 'simple-git';
+import { simpleGit } from 'simple-git';
+import * as github from '../lib/source-handlers/github';
+import type { RepoMetaData } from './types';
+import { SupportedIntegrationTypesUpdateProject } from './types';
+
+const debug = debugLib('snyk:git-clone');
+
+const urlGenerators = {
+  [SupportedIntegrationTypesUpdateProject.GITHUB]: github.buildGitCloneUrl,
+};
+
+interface GitCloneResponse {
+  success: boolean;
+  repoPath?: string;
+  gitResponse: string;
+}
+export async function gitClone(
+  integrationType: SupportedIntegrationTypesUpdateProject.GITHUB,
+  meta: RepoMetaData,
+): Promise<GitCloneResponse> {
+  const repoClonePath = await fs.mkdtempSync(
+    path.join(os.tmpdir(), `snyk-clone-${Date.now()}-${Math.random()}`),
+  );
+  try {
+    const cloneUrl = urlGenerators[integrationType](meta);
+    const options: Partial<SimpleGitOptions> = {
+      baseDir: repoClonePath,
+      binary: 'git',
+      maxConcurrentProcesses: 6,
+      trimmed: false,
+    };
+    debug(`Trying to shallow clone repo: ${meta.cloneUrl}`);
+    const git = simpleGit(options);
+    const output = await git.clone(cloneUrl, repoClonePath, {
+      '--depth': '1',
+      '--branch': meta.branch,
+    });
+
+    debug(`Repo ${meta.cloneUrl} was cloned`);
+    return {
+      gitResponse: output,
+      success: true,
+      repoPath: repoClonePath,
+    };
+  } catch (err: any) {
+    debug(`Could not shallow clone the repo:\n ${err}`);
+    if (fs.existsSync(repoClonePath)) {
+      fs.rmdirSync(repoClonePath);
+    }
+    return {
+      success: false,
+      gitResponse: err.message,
+    };
+  }
+}

src/lib/index.ts

@@ -12,6 +12,7 @@ export * from './get-snyk-host';
 export * from './filter-out-existing-orgs';
 export * from './supported-project-types';
 export * from './find-files';
+export * from './git-clone';
 
 export * from './source-handlers/github';
 export * from './source-handlers/gitlab';

src/lib/source-handlers/github/git-clone-url.ts

@@ -0,0 +1,8 @@
+import type { RepoMetaData } from '../../types';
+import { getGithubToken } from './get-github-token';
+
+export function buildGitCloneUrl(meta: RepoMetaData): string {
+  const { cloneUrl } = meta;
+  const url = new URL(cloneUrl);
+  return `${url.protocol}//${getGithubToken()}@${url.hostname}${url.pathname}`;
+}

src/lib/source-handlers/github/index.ts

@@ -4,3 +4,4 @@ export * from './organization-is-empty';
 export * from './get-repo-metadata';
 export * from './types';
 export * from './is-configured';
+export * from './git-clone-url';

test/lib/git-clone.spec.ts

@@ -0,0 +1,65 @@
+import * as fs from 'fs';
+import { gitClone } from '../../src/lib';
+import { SupportedIntegrationTypesUpdateProject } from '../../src/lib/types';
+
+describe('gitClone', () => {
+  const OLD_ENV = process.env;
+  const removeFolders: string[] = [];
+  afterAll(() => {
+    process.env = { ...OLD_ENV };
+  });
+
+  afterEach(() => {
+    for (const f of removeFolders) {
+      try {
+        fs.rmdirSync(f, { recursive: true });
+      } catch (e) {
+        console.log('Failed to clean up test', e);
+      }
+    }
+  });
+  describe('Github', () => {
+    it('successfully clones a repo', async () => {
+      process.env.GITHUB_TOKEN = process.env.GH_TOKEN;
+      process.env.SNYK_LOG_PATH = __dirname;
+
+      const res = await gitClone(
+        SupportedIntegrationTypesUpdateProject.GITHUB,
+        {
+          branch: 'master',
+          cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git',
+          sshUrl: '[email protected]:snyk-fixtures/monorepo-simple.git',
+        },
+      );
+
+      expect(res).toEqual({
+        gitResponse: '',
+        repoPath: expect.any(String),
+        success: true,
+      });
+      removeFolders.push(res.repoPath!);
+    }, 70000);
+
+    it('fails to clone a repo for non-existent branch', async () => {
+      process.env.GITHUB_TOKEN = process.env.GH_TOKEN;
+      process.env.SNYK_LOG_PATH = __dirname;
+
+      const res = await gitClone(
+        SupportedIntegrationTypesUpdateProject.GITHUB,
+        {
+          branch: 'non-existent',
+          cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git',
+          sshUrl: '[email protected]:snyk-fixtures/monorepo-simple.git',
+        },
+      );
+
+      expect(res).toEqual({
+        gitResponse: expect.stringContaining(
+          'Remote branch non-existent not found in upstream origin',
+        ),
+        success: false,
+      });
+      removeFolders.push(res.repoPath!);
+    }, 70000);
+  });
+});

test/lib/source-handlers/github/github.test.ts

@@ -80,3 +80,26 @@ describe('isGithubConfigured', () => {
     expect(() => github.isGithubConfigured()).toThrow();
   });
 });
+
+describe('buildGitCloneUrl', () => {
+  const OLD_ENV = process.env;
+
+  beforeEach(async () => {
+    delete process.env.GITHUB_TOKEN;
+  });
+
+  afterEach(async () => {
+    process.env = { ...OLD_ENV };
+  });
+  it('builds correct clone url for github.com / ghe (the urls come back from API already correct)', async () => {
+    process.env.GITHUB_TOKEN = 'secret_token';
+    const url = github.buildGitCloneUrl({
+      branch: 'main',
+      sshUrl: 'https://[email protected]:snyk-tech-services/snyk-api-import.git',
+      cloneUrl: 'https://github.com/snyk-tech-services/snyk-api-import.git',
+    });
+    expect(url).toEqual(
+      `https://[email protected]/snyk-tech-services/snyk-api-import.git`,
+    );
+  });
+});