From f70a76a8b58432b1b4fdde0c8062d80616d1e6b7 Mon Sep 17 00:00:00 2001 From: ghe Date: Tue, 6 Dec 2022 11:03:14 +0000 Subject: [PATCH] feat: clones & analyses repo for changes vs monitored projects --- .../supported-manifests.ts | 12 +- src/scripts/sync/clone-and-analyze.ts | 77 +++ .../sync/generate-projects-diff-actions.ts | 6 +- test/scripts/sync/clone-and-analyze.spec.ts | 438 ++++++++++++++++++ test/system/__snapshots__/sync.test.ts.snap | 22 - test/system/sync.test.ts | 22 +- 6 files changed, 549 insertions(+), 28 deletions(-) create mode 100644 src/scripts/sync/clone-and-analyze.ts create mode 100644 test/scripts/sync/clone-and-analyze.spec.ts delete mode 100644 test/system/__snapshots__/sync.test.ts.snap diff --git a/src/lib/supported-project-types/supported-manifests.ts b/src/lib/supported-project-types/supported-manifests.ts index e29fcf53..c869bd92 100644 --- a/src/lib/supported-project-types/supported-manifests.ts +++ b/src/lib/supported-project-types/supported-manifests.ts @@ -1,8 +1,12 @@ +export type SnykProductEntitlement = + | 'dockerfileFromScm' + | 'infrastructureAsCode'; + export const PACKAGE_MANAGERS: { [projectType: string]: { manifestFiles: string[]; isSupported: boolean; - entitlement?: string; + entitlement?: SnykProductEntitlement; }; } = { npm: { @@ -86,7 +90,7 @@ export const PACKAGE_MANAGERS: { '*[dD][oO][cC][kK][eE][rR][fF][iI][lL][eE]*', '*Dockerfile*', ], - entitlement: 'dockerfileFromScm', // TODO: use API to check https://snyk.docs.apiary.io/#reference/entitlements/a-specific-entitlement-by-organization/get-an-organization's-entitlement-value + entitlement: 'dockerfileFromScm', }, hex: { manifestFiles: ['mix.exs'], @@ -98,7 +102,7 @@ export const CLOUD_CONFIGS: { [projectType: string]: { manifestFiles: string[]; isSupported: boolean; - entitlement?: string; + entitlement?: SnykProductEntitlement; }; } = { helmconfig: { @@ -120,7 +124,7 @@ export const CLOUD_CONFIGS: { export function getSCMSupportedManifests( manifestTypes?: string[], - orgEntitlements: string[] = [], + orgEntitlements: SnykProductEntitlement[] = [], ): string[] { const typesWithSCMSupport = Object.entries({ ...PACKAGE_MANAGERS, diff --git a/src/scripts/sync/clone-and-analyze.ts b/src/scripts/sync/clone-and-analyze.ts new file mode 100644 index 00000000..c1c81beb --- /dev/null +++ b/src/scripts/sync/clone-and-analyze.ts @@ -0,0 +1,77 @@ +import * as debugLib from 'debug'; +import * as fs from 'fs'; +import * as path from 'path'; + +import { find, getSCMSupportedManifests, gitClone } from '../../lib'; +import type { SnykProductEntitlement } from '../../lib/supported-project-types/supported-manifests'; +import type { + RepoMetaData, + SnykProject, + SupportedIntegrationTypesUpdateProject, +} from '../../lib/types'; +import { generateProjectDiffActions } from './generate-projects-diff-actions'; + +const debug = debugLib('snyk:clone-and-analyze'); + +const defaultExclusionGlobs = [ + 'fixtures', + 'tests', + '__tests__', + 'test', + '__test__', + 'ci', + 'node_modules', + 'bower_components', + '.git', +]; +export async function cloneAndAnalyze( + integrationType: SupportedIntegrationTypesUpdateProject, + repoMetadata: RepoMetaData, + snykMonitoredProjects: SnykProject[], + exclusionGlobs: string[] = [], + entitlements: SnykProductEntitlement[] = [ + 'dockerfileFromScm', + 'infrastructureAsCode', + ], + manifestTypes?: string[], +): Promise<{ + import: string[]; + deactivate: SnykProject[]; +}> { + const { success, repoPath, gitResponse } = await gitClone( + integrationType, + repoMetadata, + ); + debug('Clone response', { success, repoPath, gitResponse }); + + if (!success) { + throw new Error(gitResponse); + } + + if (!repoPath) { + throw new Error('No location returned for clones repo to analyze'); + } + const { files } = await find( + repoPath, + [...defaultExclusionGlobs, ...exclusionGlobs], + // TODO: when possible switch to check entitlements via API automatically for an org + // right now the product entitlements are not exposed via API so user has to provide which products + // they are using + getSCMSupportedManifests(manifestTypes, entitlements), + 6, + ); + const relativeFileNames = files.map((f) => path.relative(repoPath, f)); + debug( + `Detected ${files.length} files in ${repoMetadata.cloneUrl}: ${files.join( + ',', + )}`, + ); + + fs.rmdirSync(repoPath, { recursive: true }); + + return generateProjectDiffActions( + relativeFileNames, + snykMonitoredProjects, + manifestTypes && manifestTypes.length > 0, + ); +} diff --git a/src/scripts/sync/generate-projects-diff-actions.ts b/src/scripts/sync/generate-projects-diff-actions.ts index ded81d4a..d0cca1df 100644 --- a/src/scripts/sync/generate-projects-diff-actions.ts +++ b/src/scripts/sync/generate-projects-diff-actions.ts @@ -3,6 +3,7 @@ import type { SnykProject } from '../../lib/types'; export function generateProjectDiffActions( repoManifests: string[], snykMonitoredProjects: SnykProject[], + skipDeactivating = false, ): { import: string[]; deactivate: SnykProject[]; @@ -29,5 +30,8 @@ export function generateProjectDiffActions( } } - return { import: filesToImport, deactivate }; + return { + import: filesToImport, + deactivate: skipDeactivating ? [] : deactivate, + }; } diff --git a/test/scripts/sync/clone-and-analyze.spec.ts b/test/scripts/sync/clone-and-analyze.spec.ts new file mode 100644 index 00000000..d68f96c6 --- /dev/null +++ b/test/scripts/sync/clone-and-analyze.spec.ts @@ -0,0 +1,438 @@ +import * as fs from 'fs'; + +import { cloneAndAnalyze } from '../../../src/scripts/sync/clone-and-analyze'; +import type { RepoMetaData, SnykProject } from '../../../src/lib/types'; + +import { SupportedIntegrationTypesUpdateProject } from '../../../src/lib/types'; + +describe('cloneAndAnalyze', () => { + const OLD_ENV = process.env; + const removeFolders: string[] = []; + afterEach(() => { + process.env = { ...OLD_ENV }; + }); + + afterEach(() => { + for (const f of removeFolders) { + try { + fs.rmdirSync(f, { recursive: true }); + } catch (e) { + console.log('Failed to clean up test', e); + } + } + }); + describe('Github', () => { + process.env.GITHUB_TOKEN = process.env.GH_TOKEN; + process.env.SNYK_LOG_PATH = __dirname; + + it('identifies correctly the diff between files in the repo vs monitored in Snyk', async () => { + // Arrange + const projects: SnykProject[] = [ + { + name: 'snyk-fixtures/monorepo-simple:package.json', + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'npm', + branch: 'master', + }, + { + name: 'snyk-fixtures/monorepo-simple:no-vulns-bundler-app/Gemfile.lock', + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'rubygems', + branch: 'master', + }, + { + name: 'snyk-fixtures/monorepo-simple:npm-project/package.json', + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'rubygems', + branch: 'master', + }, + { + name: 'snyk-fixtures/monorepo-simple:npm-project-with-policy/package.json', + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'rubygems', + branch: 'master', + }, + ]; + + const repoMeta: RepoMetaData = { + branch: 'master', + cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git', + sshUrl: 'git@github.com:snyk-fixtures/monorepo-simple.git', + }; + // Act + const res = await cloneAndAnalyze( + SupportedIntegrationTypesUpdateProject.GITHUB, + repoMeta, + projects, + undefined, + [], + undefined, + ); + + // Assert + expect(res).toStrictEqual({ + import: ['Gemfile.lock', 'bundler-app/Gemfile.lock'], + deactivate: [], + }); + }); + it('identifies correctly the diff between files in the repo vs monitored in Snyk (with IAC enabled)', async () => { + // Arrange + const projects: SnykProject[] = [ + { + name: 'snyk-fixtures/monorepo-simple:package.json', + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'npm', + branch: 'master', + }, + { + name: 'snyk-fixtures/monorepo-simple:no-vulns-bundler-app/Gemfile.lock', + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'rubygems', + branch: 'master', + }, + { + name: 'snyk-fixtures/monorepo-simple:npm-project/package.json', + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'rubygems', + branch: 'master', + }, + { + name: 'snyk-fixtures/monorepo-simple:npm-project-with-policy/package.json', + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'rubygems', + branch: 'master', + }, + ]; + + const repoMeta: RepoMetaData = { + branch: 'master', + cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git', + sshUrl: 'git@github.com:snyk-fixtures/monorepo-simple.git', + }; + // Act + const res = await cloneAndAnalyze( + SupportedIntegrationTypesUpdateProject.GITHUB, + repoMeta, + projects, + undefined, + ['infrastructureAsCode'], + undefined, + ); + + // Assert + expect(res).toStrictEqual({ + import: [ + 'Gemfile.lock', + 'bundler-app/Gemfile.lock', + 'package-2.file.json', + ], + deactivate: [], + }); + }); + it('detects new files only for a particular ecosystem when asked (npm)', async () => { + // Arrange + const projects: SnykProject[] = [ + { + name: 'snyk-fixtures/monorepo-simple:package.json', + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'npm', + branch: 'master', + }, + { + name: 'snyk-fixtures/monorepo-simple:no-vulns-bundler-app/Gemfile.lock', + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'rubygems', + branch: 'master', + }, + // Should find this file needs bringing in + // { + // name: 'snyk-fixtures/monorepo-simple:npm-project/package.json', + // id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + // created: '2018-10-29T09:50:54.014Z', + // origin: 'github', + // type: 'rubygems', + // branch: 'master', + // }, + { + name: 'snyk-fixtures/monorepo-simple:npm-project-with-policy/package.json', + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'npm', + branch: 'master', + }, + // TODO: eventually add support to identify this project needs to be de-activated + // as the file is not in repo + { + name: 'snyk-fixtures/monorepo-simple:not-in-repo/package.json', + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'npm', + branch: 'master', + }, + ]; + + const repoMeta: RepoMetaData = { + branch: 'master', + cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git', + sshUrl: 'git@github.com:snyk-fixtures/monorepo-simple.git', + }; + // Act + const res = await cloneAndAnalyze( + SupportedIntegrationTypesUpdateProject.GITHUB, + repoMeta, + projects, + undefined, + [], + ['npm'], + ); + + // Assert + // only detects which npm project need to be brought in + expect(res).toStrictEqual({ + import: ['npm-project/package.json'], + deactivate: [], + }); + }); + it('no changes needed', async () => { + // Arrange + const projects: SnykProject[] = [ + { + name: 'snyk-fixtures/monorepo-simple:package.json', + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'npm', + branch: 'master', + }, + { + name: 'snyk-fixtures/monorepo-simple:Gemfile.lock', + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'rubygems', + branch: 'master', + }, + { + name: 'snyk-fixtures/monorepo-simple:bundler-app/Gemfile.lock', + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'rubygems', + branch: 'master', + }, + { + name: 'snyk-fixtures/monorepo-simple:no-vulns-bundler-app/Gemfile.lock', + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'rubygems', + branch: 'master', + }, + { + name: 'snyk-fixtures/monorepo-simple:npm-project/package.json', + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'rubygems', + branch: 'master', + }, + { + name: 'snyk-fixtures/monorepo-simple:npm-project-with-policy/package.json', + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'rubygems', + branch: 'master', + }, + ]; + + const repoMeta: RepoMetaData = { + branch: 'master', + cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git', + sshUrl: 'git@github.com:snyk-fixtures/monorepo-simple.git', + }; + // Act + const res = await cloneAndAnalyze( + SupportedIntegrationTypesUpdateProject.GITHUB, + repoMeta, + projects, + undefined, + [], + undefined, + ); + + // Assert + expect(res).toStrictEqual({ + import: [], + deactivate: [], + }); + }, 70000); + it('processing repository with no supported manifests > nothing to do', async () => { + // Arrange + const projects: SnykProject[] = []; + + const repoMeta: RepoMetaData = { + branch: 'main', + cloneUrl: 'https://github.com/snyk-fixtures/no-supported-manifests.git', + sshUrl: 'git@github.com:snyk-fixtures/no-supported-manifests.git', + }; + + // Act & Assert + const res = await cloneAndAnalyze( + SupportedIntegrationTypesUpdateProject.GITHUB, + repoMeta, + projects, + undefined, + [], + undefined, + ); + + expect(res).toStrictEqual({ + import: [], + deactivate: [], + }); + }, 70000); + it('processing empty repository with no branch throws', async () => { + // Arrange + const projects: SnykProject[] = []; + + const repoMeta: RepoMetaData = { + branch: 'master', + cloneUrl: 'https://github.com/snyk-fixtures/empty-repo.git', + sshUrl: 'git@github.com:snyk-fixtures/empty-repo.git', + }; + + // Act & Assert + await expect( + cloneAndAnalyze( + SupportedIntegrationTypesUpdateProject.GITHUB, + repoMeta, + projects, + undefined, + [], + undefined, + ), + ).rejects.toThrowError( + 'fatal: Remote branch master not found in upstream origin', + ); + }, 70000); + }); + describe('Github Enterprise', () => { + const GHE_URL = new URL(process.env.TEST_GHE_URL!); + + afterAll(() => { + process.env = { ...OLD_ENV }; + }); + beforeEach(() => { + process.env.GITHUB_TOKEN = process.env.TEST_GHE_TOKEN; + }); + it('identifies correctly that all files need importing since the target is empty', async () => { + // Arrange + const projects: SnykProject[] = []; + + const repoMeta: RepoMetaData = { + branch: 'master', + cloneUrl: `https://${GHE_URL.host}/snyk-fixtures/mono-repo.git`, + sshUrl: `git@${GHE_URL.host}/snyk-fixtures/mono-repo.git`, + }; + // Act + const res = await cloneAndAnalyze( + SupportedIntegrationTypesUpdateProject.GITHUB, + repoMeta, + projects, + undefined, + [], + undefined, + ); + + // Assert + expect(res).toStrictEqual({ + import: [ + 'Gemfile.lock', + 'build.gradle', + 'build.sbt', + 'multi-module/pom.xml', + 'multi-module/server/pom.xml', + 'multi-module/webapp/pom.xml', + 'package.json', + 'pom.xml', + 'requirements.txt', + 'single-module/pom.xml', + ], + deactivate: [], + }); + }); + it('identifies correctly the diff between files in the repo vs monitored in Snyk (with IAC & Docker enabled)', async () => { + // Arrange + const projects: SnykProject[] = []; + + const repoMeta: RepoMetaData = { + branch: 'master', + cloneUrl: `https://${GHE_URL.host}/snyk-fixtures/docker-goof.git`, + sshUrl: `git@${GHE_URL.host}/snyk-fixtures/docker-goof.git`, + }; + // Act + const res = await cloneAndAnalyze( + SupportedIntegrationTypesUpdateProject.GITHUB, + repoMeta, + projects, + undefined, + ['infrastructureAsCode', 'dockerfileFromScm'], + undefined, + ); + + // Assert + expect(res).toStrictEqual({ + import: ['Dockerfile'], + deactivate: [], + }); + }); + it('repo appears empty when entitlements not enabled', async () => { + // Arrange + const projects: SnykProject[] = []; + + const repoMeta: RepoMetaData = { + branch: 'master', + cloneUrl: `https://${GHE_URL.host}/snyk-fixtures/docker-goof.git`, + sshUrl: `git@${GHE_URL.host}/snyk-fixtures/docker-goof.git`, + }; + // Act + const res = await cloneAndAnalyze( + SupportedIntegrationTypesUpdateProject.GITHUB, + repoMeta, + projects, + undefined, + [], + undefined, + ); + + // Assert + expect(res).toStrictEqual({ + import: [], + deactivate: [], + }); + }); + }); +}); diff --git a/test/system/__snapshots__/sync.test.ts.snap b/test/system/__snapshots__/sync.test.ts.snap deleted file mode 100644 index d77ee7e4..00000000 --- a/test/system/__snapshots__/sync.test.ts.snap +++ /dev/null @@ -1,22 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`\`snyk-api-import help <...>\` Shows help text as expected 1`] = ` -"index.js sync - -Sync targets (e.g. repos) and their projects between Snyk and SCM for a given -organization. Actions include: -- updating monitored branch in Snyk to match the default branch from SCM - -Options: - --version Show version number [boolean] - --help Show help [boolean] - --orgPublicId Public id of the organization in Snyk that will be updated - [required] - --sourceUrl Custom base url for the source API that can list organizations - (e.g. Github Enterprise url) - --source List of sources to be synced e.g. Github, Github Enterprise, - Gitlab, Bitbucket Server, Bitbucket Cloud - [required] [choices: \\"github\\", \\"github-enterprise\\"] [default: \\"github\\"] - --dryRun Dry run option. Will create a log file listing the potential - updates [default: false]" -`; diff --git a/test/system/sync.test.ts b/test/system/sync.test.ts index 89d5ddd5..8f9690ce 100644 --- a/test/system/sync.test.ts +++ b/test/system/sync.test.ts @@ -18,7 +18,27 @@ describe('`snyk-api-import help <...>`', () => { throw err; } expect(err).toBeNull(); - expect(stdout.trim()).toMatchSnapshot(); + expect(stdout).toMatchInlineSnapshot(` + "index.js sync + + Sync targets (e.g. repos) and their projects between Snyk and SCM for a given + organization. Actions include: + - updating monitored branch in Snyk to match the default branch from SCM + + Options: + --version Show version number [boolean] + --help Show help [boolean] + --orgPublicId Public id of the organization in Snyk that will be updated + [required] + --sourceUrl Custom base url for the source API that can list organizations + (e.g. Github Enterprise url) + --source List of sources to be synced e.g. Github, Github Enterprise, + Gitlab, Bitbucket Server, Bitbucket Cloud + [required] [choices: \\"github\\", \\"github-enterprise\\"] [default: \\"github\\"] + --dryRun Dry run option. Will create a log file listing the potential + updates [default: false] + " + `); }).on('exit', (code) => { expect(code).toEqual(0); done();