diff --git a/docs/import.md b/docs/import.md index 35625fcd..6f61aa1e 100644 --- a/docs/import.md +++ b/docs/import.md @@ -187,7 +187,7 @@ If you have any tests ot fixtures that should be ignored, please set the `exclus - `SNYK_LOG_PATH` - the path to folder where all logs should be saved,it is recommended creating a dedicated logs folder per import you have running. (Note: all logs will append) - `CONCURRENT_IMPORTS` (optional) defaults to 15 repos at a time, which is the recommended amount to import at once as a max. Just 1 repo may have many projects inside which can trigger a many files at once to be requested from the user's SCM instance and some may have rate limiting in place. This script aims to help reduce the risk of hitting a rate limit. - `SNYK_API` (optional) defaults to `https://snyk.io/api/v1` -- `SNYK_API_V3` (optional) defaults to `https://api.snyk.io/rest` +- `SNYK_API_REST` (optional) defaults to `https://api.snyk.io/rest` ## 3. Download & run diff --git a/package.json b/package.json index 975568ae..71ef063c 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "p-map": "4.0.0", "parse-link-header": "2.0.0", "sleep-promise": "8.0.1", - "snyk-request-manager": "1.6.0", + "snyk-request-manager": "1.7.1", "source-map-support": "^0.5.16", "split": "1.0.1", "yargs": "16.2.0" diff --git a/src/lib/api/org/index.ts b/src/lib/api/org/index.ts index e1366c4f..d98b06c7 100644 --- a/src/lib/api/org/index.ts +++ b/src/lib/api/org/index.ts @@ -166,6 +166,7 @@ interface ProjectsFilters { } export async function listProjects( + requestManager: requestsManager, orgId: string, filters?: ProjectsFilters, ): Promise { @@ -180,18 +181,12 @@ export async function listProjects( ); } - const oldsnykApiVariable = process.env.SNYK_API - process.env.SNYK_API = 'https://api.snyk.io/rest' - - const requestManager = new requestsManager({ - userAgentPrefix: 'snyk-api-import', - }); - try { const res = await requestManager.request({ - verb: 'post', - url: `/orgs/${orgId.trim()}/projects?version=2021-06-04`, + verb: 'get', + url: `/orgs/${orgId.trim()}/projects?version=2022-06-08~beta`, body: JSON.stringify(filters), + useRESTApi: true, }); const statusCode = res.statusCode || res.status; @@ -202,15 +197,15 @@ export async function listProjects( ); } - let v3responseData = res.data as v3ProjectsResponse - let projectData = convertToSnykProject(v3responseData) + let projectData = convertToSnykProject(orgId.trim(), v3responseData) while (v3responseData.links.next) { const res = await requestManager.request({ - verb: 'post', + verb: 'get', url: v3responseData.links.next, body: JSON.stringify(filters), + useRESTApi: true, }); const statusCode = res.statusCode || res.status; @@ -221,26 +216,25 @@ export async function listProjects( ); } v3responseData = res.data as v3ProjectsResponse - projectData = convertToSnykProject(res.data, projectData) + const projectDataTmp = projectData + projectData = convertToSnykProject(orgId.trim(), res.data, projectDataTmp) } - process.env.SNYK_API = oldsnykApiVariable return projectData || {}; } catch (e) { - process.env.SNYK_API = oldsnykApiVariable debug('Failed to update notification settings for ', orgId, e); throw e; } } -export function convertToSnykProject(v3Response: v3ProjectsResponse, v3PreviousDataConverted?: ProjectsResponse): ProjectsResponse { +export function convertToSnykProject(orgId: string, v3Response: v3ProjectsResponse, v3PreviousDataConverted?: ProjectsResponse): ProjectsResponse { let snykProjectData = {} as ProjectsResponse snykProjectData = { org: { name: "", - id: v3Response.data[0].relationships.org.data.id, + id: orgId, }, projects: getProjectData(v3Response.data, v3PreviousDataConverted) } @@ -251,23 +245,23 @@ export function convertToSnykProject(v3Response: v3ProjectsResponse, v3PreviousD function getProjectData(projectData: v3ProjectData[], v3PreviousDataConverted?: ProjectsResponse) : SnykProject[] { let projects = [] as SnykProject[] - const projectTmp = {} as SnykProject - - if (v3PreviousDataConverted) { - projects = v3PreviousDataConverted.projects - } - - projectData.forEach( project => { + const projectsTmp = [] as SnykProject[] - projectTmp.id = project.id + for (const project of projectData) { + const projectTmp = {} as SnykProject + projectTmp.id = project.id ? project.id : "" projectTmp.branch = project.attributes.targetReference ? project.attributes.targetReference : null projectTmp.created = project.attributes.created projectTmp.origin = project.attributes.origin projectTmp.name = project.attributes.name - projectTmp.importingUser.id = project.relationships.importingUser.data.id - - projects.push(projectTmp) - }) + projectsTmp.push(projectTmp) + } + if (v3PreviousDataConverted) { + projects = [...v3PreviousDataConverted.projects, ...projectsTmp] + } else { + projects = projectsTmp + } + return projects } diff --git a/src/scripts/generate-imported-targets-from-snyk.ts b/src/scripts/generate-imported-targets-from-snyk.ts index efc4f255..b72946e5 100644 --- a/src/scripts/generate-imported-targets-from-snyk.ts +++ b/src/scripts/generate-imported-targets-from-snyk.ts @@ -125,10 +125,9 @@ export async function generateSnykImportedTargets( const { id: orgId, name, slug } = org; try { const [resProjects, resIntegrations] = await Promise.all([ - listProjects(orgId, projectFilters), + listProjects(requestManager, orgId, projectFilters), listIntegrations(requestManager, orgId), ]); - const { projects } = resProjects; const scmTargets = projects .filter((p) => diff --git a/test/scripts/generate-imported-targets-from-snyk.test.ts b/test/scripts/generate-imported-targets-from-snyk.test.ts index 0ec10b3b..3b728337 100644 --- a/test/scripts/generate-imported-targets-from-snyk.test.ts +++ b/test/scripts/generate-imported-targets-from-snyk.test.ts @@ -91,7 +91,7 @@ describe('Generate imported targets based on Snyk data', () => { expect(importedTargetsLog).toMatch(targets[0].integrationId); }, 240000); - it.only('succeeds to generate targets for Org + Azure', async () => { + it('succeeds to generate targets for Org + Azure', async () => { const logFiles = generateLogsPaths(__dirname, ORG_ID); logs = Object.values(logFiles); const { diff --git a/test/scripts/github-enterprise-import-targets.json b/test/scripts/github-enterprise-import-targets.json new file mode 100644 index 00000000..2fcaad8a --- /dev/null +++ b/test/scripts/github-enterprise-import-targets.json @@ -0,0 +1 @@ +{"targets":[{"target":{"fork":false,"name":"goof","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"shallow-goof","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"ruby-app-rails-3","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"maven-simple-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"fixture-maven-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"goof-readers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"goof-writers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"goof-admins","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple-readers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple-writers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple-admins","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"mono-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"python-pip-app-with-vulns","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"python-pip3-app-no-policy","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"python-pip3-app","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"nested-manifest-files-only","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"goof-private","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"dotnet-project-json","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"fixture-big-file","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"dotnet-multiple-frameworks","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"js-multi-level-mono-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"js-pull-requests","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-missing-package-prop","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-no-proj-name","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-not-vuln-project","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-vuln-project","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"php-composer-lockfiles","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"docker-goof","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"}]} \ No newline at end of file diff --git a/test/system/list:imported.test.ts b/test/system/list:imported.test.ts index 0103aad6..d45b09b6 100644 --- a/test/system/list:imported.test.ts +++ b/test/system/list:imported.test.ts @@ -61,6 +61,7 @@ describe('`snyk-api-import list:imported <...>`', () => { SNYK_TOKEN: process.env.SNYK_TOKEN_TEST, SNYK_API: process.env.SNYK_API_TEST, SNYK_LOG_PATH: __dirname, + SNYK_API_REST: process.env.SNYK_API_REST, }, }, (err, stdout, stderr) => { @@ -92,6 +93,7 @@ describe('`snyk-api-import list:imported <...>`', () => { SNYK_TOKEN: process.env.SNYK_TOKEN_TEST, SNYK_API: process.env.SNYK_API_TEST, SNYK_LOG_PATH: __dirname, + SNYK_API_REST: process.env.SNYK_API_REST, }, }, (err, stdout, stderr) => { @@ -123,6 +125,7 @@ describe('`snyk-api-import list:imported <...>`', () => { SNYK_TOKEN: process.env.SNYK_TOKEN_TEST, SNYK_API: process.env.SNYK_API_TEST, SNYK_LOG_PATH: __dirname, + SNYK_API_REST: process.env.SNYK_API_REST, }, }, (err, stdout, stderr) => { @@ -154,6 +157,7 @@ describe('`snyk-api-import list:imported <...>`', () => { SNYK_TOKEN: process.env.SNYK_TOKEN_TEST, SNYK_API: process.env.SNYK_API_TEST, SNYK_LOG_PATH: __dirname, + SNYK_API_REST: process.env.SNYK_API_REST, }, }, (err, stdout, stderr) => { @@ -184,6 +188,7 @@ describe('`snyk-api-import list:imported <...>`', () => { SNYK_TOKEN: process.env.SNYK_TOKEN_TEST, SNYK_API: process.env.SNYK_API_TEST, SNYK_LOG_PATH: __dirname, + SNYK_API_REST: process.env.SNYK_API_REST, }, }, (err, stdout, stderr) => { @@ -215,6 +220,7 @@ describe('`snyk-api-import list:imported <...>`', () => { SNYK_TOKEN: process.env.SNYK_TOKEN_TEST, SNYK_API: process.env.SNYK_API_TEST, SNYK_LOG_PATH: __dirname, + SNYK_API_REST: process.env.SNYK_API_REST, }, }, (err, stdout, stderr) => {