From ad2fd15c3696bde699648e284a38b176fcfe5e5d Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Tue, 2 Aug 2022 05:20:14 +0000 Subject: [PATCH 1/2] fix: upgrade @gitbeaker/node from 35.6.1 to 35.7.0 Snyk has created this PR to upgrade @gitbeaker/node from 35.6.1 to 35.7.0. See this package in npm: https://www.npmjs.com/package/@gitbeaker/node See this project in Snyk: https://app.snyk.io/org/ilan.torbaty-k0k/project/cb9836b7-cf35-4e0c-8f5c-c30bbd5a74e8?utm_source=github&utm_medium=referral&page=upgrade-pr --- package.json | 4 +- src/lib/source-handlers/gitlab/list-repos.ts | 61 +++++++++++--------- src/lib/source-handlers/gitlab/types.ts | 2 +- 3 files changed, 36 insertions(+), 31 deletions(-) diff --git a/package.json b/package.json index b6f0a9b0..1943a037 100644 --- a/package.json +++ b/package.json @@ -38,8 +38,8 @@ ], "homepage": "https://github.com/snyk-tech-services/snyk-api-import#readme", "dependencies": { - "@gitbeaker/core": "35.6.1", - "@gitbeaker/node": "35.6.1", + "@gitbeaker/node": "35.7.0", + "@gitbeaker/core": "35.7.0", "@octokit/rest": "18.12.0", "@types/base-64": "^1.0.0", "base-64": "^1.0.0", diff --git a/src/lib/source-handlers/gitlab/list-repos.ts b/src/lib/source-handlers/gitlab/list-repos.ts index 0046bf76..b5833b9f 100644 --- a/src/lib/source-handlers/gitlab/list-repos.ts +++ b/src/lib/source-handlers/gitlab/list-repos.ts @@ -3,13 +3,13 @@ import * as debugLib from 'debug'; import { getToken } from './get-token'; import { getBaseUrl } from './get-base-url'; -import * as types from '@gitbeaker/core/dist/types'; +import * as gitBeakerTypes from '@gitbeaker/core'; import { GitlabRepoData } from './types'; const debug = debugLib('snyk:list-repos-script'); export async function fetchGitlabReposForPage( - client: types.Gitlab, + client: gitBeakerTypes.Gitlab, groupName: string, pageNumber = 1, perPage = 100, @@ -23,34 +23,39 @@ export async function fetchGitlabReposForPage( perPage, page: pageNumber, }; - const projects = await client.Groups.projects(groupName, params); + const projects = (await client.Groups.projects( + groupName, + params, + )) as gitBeakerTypes.Types.ProjectExtendedSchema[]; let hasNextPage; if (projects.length) { hasNextPage = true; - repoData.push( - ...projects - .filter((project) => { - const { - archived, - shared_with_groups, - default_branch, - } = project as types.Types.ProjectExtendedSchema; - if ( - archived || - !default_branch || - (shared_with_groups && shared_with_groups.length > 0) - ) { - return false; - } - return true; - }) - .map((project) => ({ - fork: !!project.forked_from_project, - name: project.path_with_namespace, - id: project.id, - branch: project.default_branch, - })), - ); + + for (const project of projects) { + const { + archived, + shared_with_groups, + default_branch, + forked_from_project, + path_with_namespace, + id, + } = project; + if ( + archived || + !default_branch || + !shared_with_groups || + (shared_with_groups && shared_with_groups.length === 0) + ) { + continue; + } + + repoData.push({ + fork: forked_from_project ? true : false, + name: path_with_namespace, + id, + branch: default_branch, + }); + } } else { hasNextPage = false; } @@ -58,7 +63,7 @@ export async function fetchGitlabReposForPage( } async function fetchAllRepos( - client: types.Gitlab, + client: gitBeakerTypes.Gitlab, groupName: string, page = 0, ): Promise { diff --git a/src/lib/source-handlers/gitlab/types.ts b/src/lib/source-handlers/gitlab/types.ts index 918c5122..245350f9 100644 --- a/src/lib/source-handlers/gitlab/types.ts +++ b/src/lib/source-handlers/gitlab/types.ts @@ -1,5 +1,5 @@ export interface GitlabRepoData { - // fork: boolean; + fork: boolean; branch: string; id: number; name: string; From ddd998871da69123ca70680e6ca9401ed0b2363e Mon Sep 17 00:00:00 2001 From: ghe Date: Mon, 8 Aug 2022 10:10:48 +0100 Subject: [PATCH 2/2] feat: enable axios timeouts --- package.json | 4 +- src/lib/api/group/index.ts | 9 +--- src/lib/source-handlers/github/list-repos.ts | 28 +++++----- src/lib/source-handlers/gitlab/list-groups.ts | 22 +++----- src/lib/source-handlers/gitlab/list-repos.ts | 52 ++++++++----------- test/scripts/generate-targets-data.test.ts | 4 +- 6 files changed, 49 insertions(+), 70 deletions(-) diff --git a/package.json b/package.json index 1943a037..ca13422d 100644 --- a/package.json +++ b/package.json @@ -38,8 +38,8 @@ ], "homepage": "https://github.com/snyk-tech-services/snyk-api-import#readme", "dependencies": { - "@gitbeaker/node": "35.7.0", "@gitbeaker/core": "35.7.0", + "@gitbeaker/node": "35.7.0", "@octokit/rest": "18.12.0", "@types/base-64": "^1.0.0", "base-64": "^1.0.0", @@ -51,7 +51,7 @@ "p-map": "4.0.0", "parse-link-header": "2.0.0", "sleep-promise": "8.0.1", - "snyk-request-manager": "1.7.1", + "snyk-request-manager": "1.8.0", "source-map-support": "^0.5.16", "split": "1.0.1", "yargs": "16.2.0" diff --git a/src/lib/api/group/index.ts b/src/lib/api/group/index.ts index 1f040afa..829ebff3 100644 --- a/src/lib/api/group/index.ts +++ b/src/lib/api/group/index.ts @@ -114,13 +114,8 @@ export async function listOrgsPerPage( page: pageNumber, }; const orgs = await listOrgs(requestManager, groupId, params); - let hasNextPage; - if (orgs.length) { - hasNextPage = true; - data.push(...orgs); - } else { - hasNextPage = false; - } + const hasNextPage = orgs.length ? true : false; + data.push(...orgs); return { orgs: data, hasNextPage }; } diff --git a/src/lib/source-handlers/github/list-repos.ts b/src/lib/source-handlers/github/list-repos.ts index 7eda020f..283b62a7 100644 --- a/src/lib/source-handlers/github/list-repos.ts +++ b/src/lib/source-handlers/github/list-repos.ts @@ -23,22 +23,18 @@ export async function fetchReposForPage( }; const res = await octokit.repos.listForOrg(params); const repos = res && res.data; - let hasNextPage; - if (repos.length) { - hasNextPage = true; - repoData.push( - ...repos - .filter((repo) => !repo.archived) - .map((repo) => ({ - fork: repo.fork, - name: repo.name, - owner: repo.owner?.login, - branch: repo.default_branch, - })), - ); - } else { - hasNextPage = false; - } + const hasNextPage = repos.length ? true : false; + repoData.push( + ...repos + .filter((repo) => !repo.archived) + .map((repo) => ({ + fork: repo.fork, + name: repo.name, + owner: repo.owner?.login, + branch: repo.default_branch, + })), + ); + return { repos: repoData, hasNextPage }; } diff --git a/src/lib/source-handlers/gitlab/list-groups.ts b/src/lib/source-handlers/gitlab/list-groups.ts index 6f3f5e73..5b4815c1 100644 --- a/src/lib/source-handlers/gitlab/list-groups.ts +++ b/src/lib/source-handlers/gitlab/list-groups.ts @@ -20,22 +20,16 @@ async function fetchOrgsForPage( perPage: 100, page: pageNumber, }; - let hasNextPage; - const orgs = await client.Groups.all(params); - if (orgs.length) { - hasNextPage = true; + const hasNextPage = orgs.length ? true : false; + orgsData.push( + ...orgs.map((org: any) => ({ + name: org.full_path, + id: org.id, + url: org.web_url, + })), + ); - orgsData.push( - ...orgs.map((org: any) => ({ - name: org.full_path, - id: org.id, - url: org.web_url, - })), - ); - } else { - hasNextPage = false; - } return { orgs: orgsData, hasNextPage, diff --git a/src/lib/source-handlers/gitlab/list-repos.ts b/src/lib/source-handlers/gitlab/list-repos.ts index b5833b9f..43324ad2 100644 --- a/src/lib/source-handlers/gitlab/list-repos.ts +++ b/src/lib/source-handlers/gitlab/list-repos.ts @@ -27,37 +27,31 @@ export async function fetchGitlabReposForPage( groupName, params, )) as gitBeakerTypes.Types.ProjectExtendedSchema[]; - let hasNextPage; - if (projects.length) { - hasNextPage = true; + const hasNextPage = projects.length ? true : false; - for (const project of projects) { - const { - archived, - shared_with_groups, - default_branch, - forked_from_project, - path_with_namespace, - id, - } = project; - if ( - archived || - !default_branch || - !shared_with_groups || - (shared_with_groups && shared_with_groups.length === 0) - ) { - continue; - } - - repoData.push({ - fork: forked_from_project ? true : false, - name: path_with_namespace, - id, - branch: default_branch, - }); + for (const project of projects) { + const { + archived, + shared_with_groups, + default_branch, + forked_from_project, + path_with_namespace, + id, + } = project; + if ( + archived || + !default_branch || + (shared_with_groups && shared_with_groups.length > 0) + ) { + continue; } - } else { - hasNextPage = false; + + repoData.push({ + fork: forked_from_project ? true : false, + name: path_with_namespace, + id, + branch: default_branch, + }); } return { repos: repoData, hasNextPage }; } diff --git a/test/scripts/generate-targets-data.test.ts b/test/scripts/generate-targets-data.test.ts index 42c95ee3..1eb812e2 100644 --- a/test/scripts/generate-targets-data.test.ts +++ b/test/scripts/generate-targets-data.test.ts @@ -17,7 +17,7 @@ describe('generateTargetsImportDataFile Github script', () => { }); it('generate Github repo data', async () => { process.env.GITHUB_TOKEN = process.env.GH_TOKEN; - filesToDelete.push(path.resolve(__dirname + '/github-import-targets.json')); + filesToDelete.push(path.resolve(__dirname + '/github-enterprise-import-targets.json')); const orgsData: CreatedOrg[] = [ { orgId: 'org-id', @@ -54,7 +54,7 @@ describe('generateTargetsImportDataFile Github script', () => { it('generate Github Enterprise repo data', async () => { process.env.GITHUB_TOKEN = process.env.TEST_GHE_TOKEN; const GHE_URL = process.env.TEST_GHE_URL; - filesToDelete.push(path.resolve(__dirname + '/github-import-targets.json')); + filesToDelete.push(path.resolve(__dirname + '/github-enterprise-import-targets.json')); const orgsData: CreatedOrg[] = [ { orgId: 'org-id',