diff --git a/src/lib/api/poll-import/index.ts b/src/lib/api/poll-import/index.ts index 22abed0e..f5c1ee50 100644 --- a/src/lib/api/poll-import/index.ts +++ b/src/lib/api/poll-import/index.ts @@ -78,7 +78,7 @@ export async function pollImportUrl( export async function pollImportUrls( requestManager: requestsManager, locationUrls: string[], -): Promise<{ projects: Project[] }> { +): Promise<{ projects: Project[]; failed: FailedProject[] }> { if (!locationUrls) { throw new Error( 'Missing required parameters. Please ensure you have provided: locationUrls.', @@ -130,5 +130,5 @@ export async function pollImportUrls( await logFailedProjects(allFailedProjects); - return { projects: projectsArray }; + return { projects: projectsArray, failed: allFailedProjects }; } diff --git a/src/lib/types.ts b/src/lib/types.ts index 751c569a..03bb59d2 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -251,3 +251,9 @@ export type SyncTargetsConfig = { manifestTypes?: string[]; exclusionGlobs?: string[]; }; + +export enum ProjectUpdateType { + BRANCH = 'branch', + DEACTIVATE = 'deactivate', + IMPORT = 'import', +} diff --git a/src/loggers/log-failed-projects.ts b/src/loggers/log-failed-projects.ts index cae28f50..2b801c42 100644 --- a/src/loggers/log-failed-projects.ts +++ b/src/loggers/log-failed-projects.ts @@ -9,6 +9,7 @@ const debug = debugLib('snyk:import-projects-script'); export interface FailedProject extends Project { locationUrl: string; + userMessage?: string; } export async function logFailedProjects( diff --git a/src/scripts/sync/import-target.ts b/src/scripts/sync/import-target.ts index b658105f..1b55c69d 100644 --- a/src/scripts/sync/import-target.ts +++ b/src/scripts/sync/import-target.ts @@ -2,11 +2,13 @@ import type { requestsManager } from 'snyk-request-manager'; import * as debugLib from 'debug'; import { defaultExclusionGlobs } from '../../common'; import { importTarget, listIntegrations, pollImportUrls } from '../../lib'; + import type { Project, - SupportedIntegrationTypesUpdateProject, Target, + SupportedIntegrationTypesUpdateProject, } from '../../lib/types'; +import type { FailedProject } from '../../loggers/log-failed-projects'; const debug = debugLib('snyk:import-single-target'); @@ -18,7 +20,7 @@ export async function importSingleTarget( filesToImport: string[] = [], excludeFolders?: string, loggingPath?: string, -): Promise<{ projects: Project[] }> { +): Promise<{ projects: Project[]; failed: FailedProject[] }> { const integrationsData = await listIntegrations(requestManager, orgId); const integrationId = integrationsData[integrationType]; const files = filesToImport.map((f) => ({ path: f })); diff --git a/src/scripts/sync/sync-projects-per-target.ts b/src/scripts/sync/sync-projects-per-target.ts index 1cd4a459..a689af40 100644 --- a/src/scripts/sync/sync-projects-per-target.ts +++ b/src/scripts/sync/sync-projects-per-target.ts @@ -10,17 +10,15 @@ import type { RepoMetaData, SyncTargetsConfig, } from '../../lib/types'; +import { ProjectUpdateType } from '../../lib/types'; import { SupportedIntegrationTypesUpdateProject } from '../../lib/types'; import { targetGenerators } from '../generate-imported-targets-from-snyk'; import { deactivateProject, listProjects } from '../../lib'; import pMap = require('p-map'); import { cloneAndAnalyze } from './clone-and-analyze'; +import { importSingleTarget } from './import-target'; const debug = debugLib('snyk:sync-projects-per-target'); -export enum ProjectUpdateType { - BRANCH = 'branch', - DEACTIVATE = 'deactivate', -} export function getMetaDataGenerator( origin: SupportedIntegrationTypesUpdateProject, ): (target: Target, host?: string | undefined) => Promise { @@ -272,3 +270,73 @@ export async function bulkDeactivateProjects( return { updated, failed }; } + +export async function bulkImportTargetFiles( + requestManager: requestsManager, + orgId: string, + files: string[] = [], + integrationType: SupportedIntegrationTypesUpdateProject, + target: Target, + dryRun = false, + concurrentFilesImport = 30, +): Promise<{ created: ProjectUpdate[]; failed: ProjectUpdateFailure[] }> { + const created: ProjectUpdate[] = []; + const failed: ProjectUpdateFailure[] = []; + + if (!files.length) { + return { created, failed }; + } + debug(`Importing ${files.length} files`); + + if (dryRun) { + files.map((f) => + created.push({ + projectPublicId: '', + type: ProjectUpdateType.IMPORT, + from: f, + to: `https://app.snyk.io/org/example-org-name/project/example-project-id-uuid`, + dryRun, + }), + ); + + return { created, failed: [] }; + } + + for ( + let index = 0; + index < files.length; + index = index + concurrentFilesImport + ) { + const batch = files.slice(index, index + concurrentFilesImport); + + const { projects, failed: failedProjects } = await importSingleTarget( + requestManager, + orgId, + integrationType, + target, + batch, + ); + projects.map((p) => { + const projectId = p.projectUrl?.split('/').slice(-1)[0]; + created.push({ + projectPublicId: projectId, + type: ProjectUpdateType.IMPORT, + from: p.targetFile ?? '', // TODO: is there something more intuitive here? + to: p.projectUrl, + dryRun, + }); + }); + failedProjects.map((f) => { + failed.push({ + projectPublicId: '', + type: ProjectUpdateType.IMPORT, + from: f.targetFile ?? '', // TODO: is there something more intuitive here? + to: f.projectUrl, + dryRun, + errorMessage: + f.userMessage ?? 'Failed to import project via Snyk Import API', + }); + }); + } + return { created, failed }; +} diff --git a/test/scripts/sync/sync-org-projects.test.ts b/test/scripts/sync/sync-org-projects.test.ts index b6377f76..de4806f0 100644 --- a/test/scripts/sync/sync-org-projects.test.ts +++ b/test/scripts/sync/sync-org-projects.test.ts @@ -7,12 +7,14 @@ import { updateTargets, } from '../../../src/scripts/sync/sync-org-projects'; import type { ProjectsResponse } from '../../../src/lib/api/org'; -import * as syncProjectsForTarget from '../../../src/scripts/sync/sync-projects-per-target'; +import type * as syncProjectsForTarget from '../../../src/scripts/sync/sync-projects-per-target'; import type { + Project, SnykProject, SnykTarget, SnykTargetRelationships, } from '../../../src/lib/types'; +import { ProjectUpdateType } from '../../../src/lib/types'; import { SupportedIntegrationTypesUpdateProject } from '../../../src/lib/types'; import * as lib from '../../../src/lib'; import * as clone from '../../../src/scripts/sync/clone-and-analyze'; @@ -20,6 +22,11 @@ import * as projectApi from '../../../src/lib/api/project'; import * as github from '../../../src/lib/source-handlers/github'; import * as featureFlags from '../../../src/lib/api/feature-flags'; import * as updateProjectsLog from '../../../src/loggers/log-updated-project'; +import * as importTarget from '../../../src/scripts/sync/import-target'; +import { deleteFiles } from '../../delete-files'; +import { generateLogsPaths } from '../../generate-log-file-names'; +import { bulkImportTargetFiles } from '../../../src/scripts/sync/sync-projects-per-target'; +import type { FailedProject } from '../../../src/loggers/log-failed-projects'; const fixturesFolderPath = path.resolve(__dirname, '../') + '/fixtures/repos'; describe('updateTargets', () => { @@ -125,7 +132,7 @@ describe('updateTargets', () => { projectPublicId: projectsAPIResponse.projects[0].id, from: projectsAPIResponse.projects[0].branch!, to: defaultBranch, - type: syncProjectsForTarget.ProjectUpdateType.BRANCH, + type: ProjectUpdateType.BRANCH, dryRun: false, target: testTargets[0], }, @@ -133,7 +140,7 @@ describe('updateTargets', () => { projectPublicId: projectsAPIResponse.projects[2].id, from: 'active', to: 'deactivated', - type: syncProjectsForTarget.ProjectUpdateType.DEACTIVATE, + type: ProjectUpdateType.DEACTIVATE, dryRun: false, target: testTargets[0], }, @@ -145,7 +152,7 @@ describe('updateTargets', () => { projectPublicId: projectsAPIResponse.projects[1].id, from: projectsAPIResponse.projects[1].branch!, to: defaultBranch, - type: syncProjectsForTarget.ProjectUpdateType.BRANCH, + type: ProjectUpdateType.BRANCH, dryRun: false, target: testTargets[0], }, @@ -255,7 +262,7 @@ describe('updateTargets', () => { projectPublicId: 'af137b96-6966-46c1-826b-2e79ac49bbxx', from: projectsAPIResponse.projects[0].branch!, to: defaultBranch, - type: syncProjectsForTarget.ProjectUpdateType.BRANCH, + type: ProjectUpdateType.BRANCH, dryRun: false, }, ]; @@ -360,7 +367,7 @@ describe('updateTargets', () => { projectPublicId: 'af137b96-6966-46c1-826b-2e79ac49bbxx', from: 'active', to: 'deactivated', - type: syncProjectsForTarget.ProjectUpdateType.DEACTIVATE, + type: ProjectUpdateType.DEACTIVATE, dryRun: false, }, ]; @@ -556,7 +563,7 @@ describe('updateTargets', () => { projectPublicId: projectsAPIResponse.projects[0].id, from: projectsAPIResponse.projects[0].branch!, to: defaultBranch, - type: syncProjectsForTarget.ProjectUpdateType.BRANCH, + type: ProjectUpdateType.BRANCH, dryRun: false, }, ]; @@ -567,7 +574,7 @@ describe('updateTargets', () => { projectPublicId: projectsAPIResponse.projects[1].id, from: projectsAPIResponse.projects[1].branch!, to: defaultBranch, - type: syncProjectsForTarget.ProjectUpdateType.BRANCH, + type: ProjectUpdateType.BRANCH, dryRun: false, }, ]; @@ -675,7 +682,7 @@ describe('updateTargets', () => { projectPublicId: projectsAPIResponse.projects[0].id, from: projectsAPIResponse.projects[0].branch!, to: defaultBranch, - type: syncProjectsForTarget.ProjectUpdateType.BRANCH, + type: ProjectUpdateType.BRANCH, dryRun: false, }, ]; @@ -686,7 +693,7 @@ describe('updateTargets', () => { projectPublicId: projectsAPIResponse.projects[1].id, from: projectsAPIResponse.projects[1].branch!, to: defaultBranch, - type: syncProjectsForTarget.ProjectUpdateType.BRANCH, + type: ProjectUpdateType.BRANCH, dryRun: false, }, ]; @@ -1072,7 +1079,7 @@ describe('updateOrgTargets', () => { projectPublicId: updatedProjectId1, from: projectsAPIResponse1.projects[0].branch!, to: defaultBranch, - type: syncProjectsForTarget.ProjectUpdateType.BRANCH, + type: ProjectUpdateType.BRANCH, dryRun: true, target: targets[0], }, @@ -1080,7 +1087,7 @@ describe('updateOrgTargets', () => { projectPublicId: deletedProjectId, from: 'active', to: 'deactivated', - type: syncProjectsForTarget.ProjectUpdateType.DEACTIVATE, + type: ProjectUpdateType.DEACTIVATE, dryRun: true, target: targets[0], }, @@ -1088,7 +1095,7 @@ describe('updateOrgTargets', () => { projectPublicId: updatedProjectId2, from: projectsAPIResponse2.projects[0].branch!, to: defaultBranch, - type: syncProjectsForTarget.ProjectUpdateType.BRANCH, + type: ProjectUpdateType.BRANCH, dryRun: true, target: targets[1], }, @@ -1263,7 +1270,7 @@ describe('updateOrgTargets', () => { projectPublicId: updatedProjectId1, from: projectsAPIResponse1.projects[0].branch!, to: defaultBranch, - type: syncProjectsForTarget.ProjectUpdateType.BRANCH, + type: ProjectUpdateType.BRANCH, dryRun: true, target: targets[0], }, @@ -1271,7 +1278,7 @@ describe('updateOrgTargets', () => { projectPublicId: updatedProjectId2, from: projectsAPIResponse2.projects[0].branch!, to: defaultBranch, - type: syncProjectsForTarget.ProjectUpdateType.BRANCH, + type: ProjectUpdateType.BRANCH, dryRun: true, target: targets[1], }, @@ -1317,3 +1324,222 @@ describe('updateOrgTargets', () => { ); }); }); + +describe('bulkImportTargetFiles', () => { + let logs: string[]; + const OLD_ENV = process.env; + process.env.SNYK_TOKEN = process.env.SNYK_TOKEN_TEST; + const ORG_ID = 'af137b96-6966-46c1-826b-2e79ac49bbxx'; + let importSingleTargetSpy: jest.SpyInstance; + + afterAll(async () => { + await deleteFiles(logs); + process.env = { ...OLD_ENV }; + }, 10000); + + const requestManager = new requestsManager({ + userAgentPrefix: 'snyk-api-import:tests', + }); + + beforeEach(() => { + importSingleTargetSpy = jest.spyOn(importTarget, 'importSingleTarget'); + }); + + afterEach(() => { + importSingleTargetSpy.mockReset(); + }); + it('succeeds to import a single file', async () => { + // Arrange + const logFiles = generateLogsPaths(__dirname, ORG_ID); + logs = Object.values(logFiles); + const target = { + name: 'ruby-with-versions', + owner: 'api-import-circle-test', + branch: 'master', + }; + const projectId = uuid.v4(); + const projects: Project[] = [ + { + projectUrl: `https://app.snyk.io/org/hello/project/${projectId}`, + success: true, + targetFile: 'Gemfile.lock', + }, + ]; + importSingleTargetSpy.mockResolvedValue({ + projects, + failed: [], + }); + + const { created } = await bulkImportTargetFiles( + requestManager, + ORG_ID, + ['ruby-2.5.3-exactly/Gemfile'], + SupportedIntegrationTypesUpdateProject.GHE, + target, + ); + expect(importSingleTargetSpy).toHaveBeenCalledTimes(1); + expect(created).not.toBe([]); + expect(created.length).toEqual(1); + expect(created[0]).toMatchObject({ + dryRun: false, + from: 'Gemfile.lock', + projectPublicId: projectId, + to: `https://app.snyk.io/org/hello/project/${projectId}`, + type: 'import', + }); + }); + it('batch imports many files to keep import jobs smaller', async () => { + // Arrange + const logFiles = generateLogsPaths(__dirname, ORG_ID); + logs = Object.values(logFiles); + const target = { + name: 'ruby-with-versions', + owner: 'api-import-circle-test', + branch: 'master', + }; + const projectId = uuid.v4(); + const projects: Project[] = [ + { + projectUrl: `https://app.snyk.io/org/hello/project/${projectId}`, + success: true, + targetFile: 'Gemfile.lock', + }, + { + projectUrl: `https://app.snyk.io/org/hello/project/${projectId}`, + success: true, + targetFile: 'package.json', + }, + { + projectUrl: `https://app.snyk.io/org/hello/project/${projectId}`, + success: true, + targetFile: 'folder/package.json', + }, + { + projectUrl: `https://app.snyk.io/org/hello/project/${projectId}`, + success: true, + targetFile: 'Dockerfile', + }, + { + projectUrl: `https://app.snyk.io/org/hello/project/${projectId}`, + success: true, + targetFile: 'another/Dockerfile', + }, + ]; + const failedProjects: FailedProject[] = [ + { + projectUrl: '', + success: false, + locationUrl: 'https://polling/url', + targetFile: 'failure/Dockerfile', + userMessage: 'Invalid syntax', + }, + ]; + + importSingleTargetSpy.mockResolvedValueOnce({ + projects: [projects[0], projects[1]], + failed: [], + }); + importSingleTargetSpy.mockResolvedValueOnce({ + projects: [projects[2], projects[3]], + failed: [], + }); + importSingleTargetSpy.mockResolvedValueOnce({ + projects: [projects[4]], + failed: [failedProjects[0]], + }); + + const { created, failed } = await bulkImportTargetFiles( + requestManager, + ORG_ID, + projects.map((p) => p.targetFile!), + SupportedIntegrationTypesUpdateProject.GHE, + target, + false, + 2, + ); + expect(importSingleTargetSpy).toHaveBeenCalledTimes(3); + expect(created).not.toBe([]); + expect(created.length).toEqual(5); + expect(failed.length).toEqual(1); + + expect(created.find((c) => c.from === 'folder/package.json')).toMatchObject( + { + dryRun: false, + from: 'folder/package.json', + projectPublicId: projectId, + to: `https://app.snyk.io/org/hello/project/${projectId}`, + type: 'import', + }, + ); + expect(failed.find((c) => c.from === 'failure/Dockerfile')).toMatchObject({ + dryRun: false, + from: 'failure/Dockerfile', + projectPublicId: '', + to: '', + type: 'import', + }); + }); + it('batch imports many files in dryRun mode', async () => { + // Arrange + const logFiles = generateLogsPaths(__dirname, ORG_ID); + logs = Object.values(logFiles); + const target = { + name: 'ruby-with-versions', + owner: 'api-import-circle-test', + branch: 'master', + }; + const projectId = uuid.v4(); + const projects: Project[] = [ + { + projectUrl: `https://app.snyk.io/org/hello/project/${projectId}`, + success: true, + targetFile: 'Gemfile.lock', + }, + { + projectUrl: `https://app.snyk.io/org/hello/project/${projectId}`, + success: true, + targetFile: 'package.json', + }, + { + projectUrl: `https://app.snyk.io/org/hello/project/${projectId}`, + success: true, + targetFile: 'folder/package.json', + }, + { + projectUrl: `https://app.snyk.io/org/hello/project/${projectId}`, + success: true, + targetFile: 'Dockerfile', + }, + { + projectUrl: `https://app.snyk.io/org/hello/project/${projectId}`, + success: true, + targetFile: 'another/Dockerfile', + }, + ]; + importSingleTargetSpy.mockResolvedValueOnce({ + projects: [projects[0], projects[1]], + failed: [], + }); + importSingleTargetSpy.mockResolvedValueOnce({ + projects: [projects[2], projects[3]], + failed: [], + }); + importSingleTargetSpy.mockResolvedValueOnce({ + projects: [projects[4]], + failed: [], + }); + + const { created } = await bulkImportTargetFiles( + requestManager, + ORG_ID, + projects.map((p) => p.targetFile!), + SupportedIntegrationTypesUpdateProject.GHE, + target, + true, + 2, + ); + expect(importSingleTargetSpy).toHaveBeenCalledTimes(0); + expect(created).not.toBe([]); + expect(created.length).toEqual(5); + }); +});