diff --git a/package.json b/package.json index 975568ae..71ef063c 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "p-map": "4.0.0", "parse-link-header": "2.0.0", "sleep-promise": "8.0.1", - "snyk-request-manager": "1.6.0", + "snyk-request-manager": "1.7.1", "source-map-support": "^0.5.16", "split": "1.0.1", "yargs": "16.2.0" diff --git a/src/lib/api/org/index.ts b/src/lib/api/org/index.ts index 0b957f93..42d3be53 100644 --- a/src/lib/api/org/index.ts +++ b/src/lib/api/org/index.ts @@ -3,7 +3,7 @@ import { requestsManager } from 'snyk-request-manager'; import * as debugLib from 'debug'; import { getApiToken } from '../../get-api-token'; import { getSnykHost } from '../../get-snyk-host'; -import { SnykProject } from '../../types'; +import { SnykProject, v3ProjectData } from '../../types'; import { StringNullableChain } from 'lodash'; const debug = debugLib('snyk:api-group'); @@ -142,6 +142,22 @@ interface ProjectsResponse { projects: SnykProject[]; } + +interface v3ProjectsResponse { + data: v3ProjectData[]; + jsonapi: { + version: string; + }; + links: { + first: string; + last: string; + next: string; + prev: string; + related: string; + self: string; + }; +} + interface ProjectsFilters { name?: StringNullableChain; // If supplied, only projects that have a name that starts with this value will be returned origin?: string; //If supplied, only projects that exactly match this origin will be returned @@ -149,6 +165,7 @@ interface ProjectsFilters { isMonitored?: boolean; // If set to true, only include projects which are monitored, if set to false, only include projects which are not monitored } + export async function listProjects( requestManager: requestsManager, orgId: string, @@ -165,23 +182,99 @@ export async function listProjects( ); } - try { - const res = await requestManager.request({ - verb: 'post', - url: `/org/${orgId.trim()}/projects`, - body: JSON.stringify(filters), - }); + let snykProjectData = {} as ProjectsResponse - const statusCode = res.statusCode || res.status; - if (!statusCode || statusCode !== 200) { - throw new Error( - 'Expected a 200 response, instead received: ' + - JSON.stringify({ data: res.data, status: statusCode }), - ); + const projects = await listAllProjects(requestManager, orgId, filters) + + snykProjectData = { + org: { + name: "", + id: orgId, + }, + projects: projects + } + + return snykProjectData; +} + +async function listAllProjects(requestManager: requestsManager, + orgId: string, + filters?: ProjectsFilters, + ): Promise { + + let lastPage = false; + let projectsList: SnykProject[] = []; + let pageCount = 1; + let nextPageLink: string | undefined = undefined; + while (!lastPage) { + debug(`Fetching page ${pageCount}\n`); + try { + const { + projects, + next, + }: { + projects: SnykProject[]; + next?: string; + } = await getProject(requestManager, orgId, filters, nextPageLink); + + projectsList = projectsList.concat(projects); + next + ? ((lastPage = false), (nextPageLink = next)) + : ((lastPage = true), (nextPageLink = '')); + pageCount++; + } catch (e) { + debug('Failed to update notification settings for ', orgId, e); + throw e; } - return res.data || {}; - } catch (e) { - debug('Failed to update notification settings for ', orgId, e); - throw e; } + return projectsList } + +async function getProject(requestManager: requestsManager, + orgId: string, + filters?: ProjectsFilters, + nextPageLink?: string, + ): Promise< { projects: SnykProject[], next?: string } > { + + const url = nextPageLink ? nextPageLink : `/orgs/${orgId.trim()}/projects?version=2022-06-08~beta` + const res = await requestManager.request({ + verb: 'get', + url: url, + body: JSON.stringify(filters), + useRESTApi: true, + }); + + const statusCode = res.statusCode || res.status; + if (!statusCode || statusCode !== 200) { + throw new Error( + 'Expected a 200 response, instead received: ' + + JSON.stringify({ data: res.data, status: statusCode }), + ); + } + + const v3responseData = res.data as v3ProjectsResponse + + const projects = convertToSnykProject(v3responseData.data) + const next = v3responseData.links.next + + return { projects, next} + +} + +function convertToSnykProject(projectData: v3ProjectData[]) : SnykProject[] { + + const projects: SnykProject[] = []; + + for (const project of projectData) { + const projectTmp = {} as SnykProject + projectTmp.id = project.id + projectTmp.branch = project.attributes.targetReference + projectTmp.created = project.attributes.created + projectTmp.origin = project.attributes.origin + projectTmp.name = project.attributes.name + projects.push(projectTmp) + } + + return projects +} + diff --git a/src/lib/types.ts b/src/lib/types.ts index 62e36436..b29a25f4 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -117,15 +117,15 @@ export interface CommandResult { } export interface SnykProject { - name: string; - id: string; - created: string; - origin: string; - type: string; - readOnly: boolean; - testFrequency: string; - isMonitored: boolean; - totalDependencies: number; + name: string; + id: string; + created: string; + origin: string; + type: string; + readOnly: boolean; + testFrequency: string; + isMonitored: boolean; + totalDependencies: number; issueCountsBySeverity: { low: number; high: number; @@ -135,17 +135,18 @@ export interface SnykProject { remoteRepoUrl: string; // URL of the repo lastTestedDate: string; browseUrl: string; - owner: string | null; - importingUser: ImportingUser; - tags: unknown[]; + owner: string | null; + importingUser: ImportingUser; + tags: unknown[]; attributes: { - criticality: unknown[]; - lifecycle: unknown[]; - environment: unknown[]; + criticality: unknown[]; + lifecycle: unknown[]; + environment: unknown[]; }; - branch: string | null; + branch: string | null; } + export interface Org { name: string; id: string; @@ -156,3 +157,40 @@ export interface Org { id: string; }; } + + +export interface v3ProjectData { + attributes: v3ProjectsAttributes; + id: string; + relationships: v3ProjectsRelationships; + type: string +} + +export interface v3ProjectsAttributes { + businessCriticality: string; + created: string; + environment: string; + lifecycle: string; + name: string; + origin: string; + status: string; + tags: unknown; + targetReference: string | null; + type: string; +} + +export interface v3ProjectsRelationships { + importingUser: v3ProjectsRelashionshipData; + org: v3ProjectsRelashionshipData; + owner: v3ProjectsRelashionshipData; + target: v3ProjectsRelashionshipData; +} + +export interface v3ProjectsRelashionshipData { + data: { + id: string; + type: string; + } + links: unknown; + meta: unknown; +} diff --git a/src/scripts/generate-imported-targets-from-snyk.ts b/src/scripts/generate-imported-targets-from-snyk.ts index b70f8624..b72946e5 100644 --- a/src/scripts/generate-imported-targets-from-snyk.ts +++ b/src/scripts/generate-imported-targets-from-snyk.ts @@ -130,20 +130,21 @@ export async function generateSnykImportedTargets( ]); const { projects } = resProjects; const scmTargets = projects - .filter((p) => - integrationTypes.includes( - p.origin as SupportedIntegrationTypesToListSnykTargets, - ), - ) - .map((p) => { - const target = targetGenerators[ - p.origin as SupportedIntegrationTypesToListSnykTargets - ](p); - return { - target, - integrationId: resIntegrations[p.origin], - }; - }); + .filter((p) => + integrationTypes.includes( + p.origin as SupportedIntegrationTypesToListSnykTargets, + ), + ) + .map((p) => { + const target = targetGenerators[ + p.origin as SupportedIntegrationTypesToListSnykTargets + ](p); + return { + target, + integrationId: resIntegrations[p.origin], + }; + }); + const uniqueTargets: Set = new Set(); const orgTargets: Target[] = []; if (!scmTargets.length || scmTargets.length === 0) { diff --git a/test/scripts/github-enterprise-import-targets.json b/test/scripts/github-enterprise-import-targets.json new file mode 100644 index 00000000..2fcaad8a --- /dev/null +++ b/test/scripts/github-enterprise-import-targets.json @@ -0,0 +1 @@ +{"targets":[{"target":{"fork":false,"name":"goof","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"shallow-goof","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"ruby-app-rails-3","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"maven-simple-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"fixture-maven-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"goof-readers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"goof-writers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"goof-admins","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple-readers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple-writers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple-admins","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"mono-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"python-pip-app-with-vulns","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"python-pip3-app-no-policy","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"python-pip3-app","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"nested-manifest-files-only","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"goof-private","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"dotnet-project-json","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"fixture-big-file","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"dotnet-multiple-frameworks","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"js-multi-level-mono-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"js-pull-requests","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-missing-package-prop","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-no-proj-name","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-not-vuln-project","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-vuln-project","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"php-composer-lockfiles","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"docker-goof","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"}]} \ No newline at end of file