diff --git a/docs/import.md b/docs/import.md index b6582bd2..35625fcd 100644 --- a/docs/import.md +++ b/docs/import.md @@ -187,6 +187,7 @@ If you have any tests ot fixtures that should be ignored, please set the `exclus - `SNYK_LOG_PATH` - the path to folder where all logs should be saved,it is recommended creating a dedicated logs folder per import you have running. (Note: all logs will append) - `CONCURRENT_IMPORTS` (optional) defaults to 15 repos at a time, which is the recommended amount to import at once as a max. Just 1 repo may have many projects inside which can trigger a many files at once to be requested from the user's SCM instance and some may have rate limiting in place. This script aims to help reduce the risk of hitting a rate limit. - `SNYK_API` (optional) defaults to `https://snyk.io/api/v1` +- `SNYK_API_V3` (optional) defaults to `https://api.snyk.io/rest` ## 3. Download & run diff --git a/src/lib/api/org/index.ts b/src/lib/api/org/index.ts index 0b957f93..aee6294f 100644 --- a/src/lib/api/org/index.ts +++ b/src/lib/api/org/index.ts @@ -3,7 +3,7 @@ import { requestsManager } from 'snyk-request-manager'; import * as debugLib from 'debug'; import { getApiToken } from '../../get-api-token'; import { getSnykHost } from '../../get-snyk-host'; -import { SnykProject } from '../../types'; +import { SnykProject, v3ProjectData } from '../../types'; import { StringNullableChain } from 'lodash'; const debug = debugLib('snyk:api-group'); @@ -142,6 +142,15 @@ interface ProjectsResponse { projects: SnykProject[]; } + +interface v3ProjectsResponse { + data: v3ProjectData[]; + jsonapi: { + version: string; + }; + links: unknown; +} + interface ProjectsFilters { name?: StringNullableChain; // If supplied, only projects that have a name that starts with this value will be returned origin?: string; //If supplied, only projects that exactly match this origin will be returned @@ -153,9 +162,9 @@ export async function listProjects( requestManager: requestsManager, orgId: string, filters?: ProjectsFilters, -): Promise { +): Promise { getApiToken(); - getSnykHost(); + const SNYK_API = getSnykHost("v3"); debug(`Listing all projects for org: ${orgId}`); if (!orgId) { @@ -168,7 +177,7 @@ export async function listProjects( try { const res = await requestManager.request({ verb: 'post', - url: `/org/${orgId.trim()}/projects`, + url: `${SNYK_API}/orgs/${orgId.trim()}/projects?version=2021-06-04`, body: JSON.stringify(filters), }); diff --git a/src/lib/get-snyk-host.ts b/src/lib/get-snyk-host.ts index 33ceac33..5ee07dc4 100644 --- a/src/lib/get-snyk-host.ts +++ b/src/lib/get-snyk-host.ts @@ -1,3 +1,8 @@ -export function getSnykHost(): string { - return process.env.SNYK_API || 'https://snyk.io/api/v1'; +export function getSnykHost(ApiVersion?: string): string { + let apiUrl = 'https://snyk.io/api/v1' + if (ApiVersion === "v3") { + apiUrl = 'https://api.snyk.io/rest' + return process.env.SNYK_API_V3 || apiUrl; + } + return process.env.SNYK_API || apiUrl; } diff --git a/src/lib/types.ts b/src/lib/types.ts index 62e36436..68b7b4fd 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -117,15 +117,15 @@ export interface CommandResult { } export interface SnykProject { - name: string; - id: string; - created: string; - origin: string; - type: string; - readOnly: boolean; - testFrequency: string; - isMonitored: boolean; - totalDependencies: number; + name: string; + id: string; + created: string; + origin: string; + type: string; + readOnly: boolean; + testFrequency: string; + isMonitored: boolean; + totalDependencies: number; issueCountsBySeverity: { low: number; high: number; @@ -135,17 +135,18 @@ export interface SnykProject { remoteRepoUrl: string; // URL of the repo lastTestedDate: string; browseUrl: string; - owner: string | null; - importingUser: ImportingUser; - tags: unknown[]; + owner: string | null; + importingUser: ImportingUser; + tags: unknown[]; attributes: { - criticality: unknown[]; - lifecycle: unknown[]; - environment: unknown[]; + criticality: unknown[]; + lifecycle: unknown[]; + environment: unknown[]; }; - branch: string | null; + branch: string | null; } + export interface Org { name: string; id: string; @@ -156,3 +157,40 @@ export interface Org { id: string; }; } + + +export interface v3ProjectData { + attributes: v3ProjectsAttributes; + id: string; + relationships: v3ProjectsRelationships; + type: string +} + +export interface v3ProjectsAttributes { + businessCriticality: string; + created: string; + environment: string; + lifecycle: string; + name: string; + origin: string; + status: string; + tags: unknown; + targetReference: string | undefined; + type: string; +} + +export interface v3ProjectsRelationships { + importingUser: v3ProjectsRelashionshipData; + org: v3ProjectsRelashionshipData; + owner: v3ProjectsRelashionshipData; + target: v3ProjectsRelashionshipData; +} + +export interface v3ProjectsRelashionshipData { + data: { + id: string; + type: string; + } + links: unknown; + meta: unknown; +} diff --git a/src/scripts/generate-imported-targets-from-snyk.ts b/src/scripts/generate-imported-targets-from-snyk.ts index b70f8624..72294717 100644 --- a/src/scripts/generate-imported-targets-from-snyk.ts +++ b/src/scripts/generate-imported-targets-from-snyk.ts @@ -6,9 +6,9 @@ import * as _ from 'lodash'; import { FilePath, - SnykProject, SupportedIntegrationTypesToListSnykTargets, Target, + v3ProjectsAttributes, } from '../lib/types'; import { getAllOrgs, @@ -31,19 +31,19 @@ export interface ImportTarget { } export function projectToTarget( - project: Pick, + projectAttributes: Pick, ): Target { - const [owner, name] = project.name.split(':')[0].split('/'); + const [owner, name] = projectAttributes.name.split(':')[0].split('/'); return { owner, - branch: project.branch || undefined, // TODO: make it not optional + branch: projectAttributes.targetReference || undefined, // TODO: make it not optional name, }; } export function bitbucketServerProjectToTarget( - project: Pick, + projectAttributes: Pick, ): Target { - const [projectKey, repoSlug] = project.name.split(':')[0].split('/'); + const [projectKey, repoSlug] = projectAttributes.name.split(':')[0].split('/'); return { projectKey, repoSlug, @@ -51,22 +51,22 @@ export function bitbucketServerProjectToTarget( } export function gitlabProjectToImportLogTarget( - project: Pick, + projectAttributes: Pick, ): Target { // Gitlab target is only `id` & branch and the Snyk API does not return the id. // However we are already logging `name` which for Gitlab is "owner/repo", branch & id so if we use the same name we can match on it - const name = project.name.split(':')[0]; + const name = projectAttributes.name.split(':')[0]; return { - branch: project.branch || undefined, // TODO: make it not optional + branch: projectAttributes.targetReference || undefined, // TODO: make it not optional name, }; } export function imageProjectToTarget( - project: Pick, + projectAttributes: Pick, ): Target { return { - name: project.name, + name: projectAttributes.name, }; } @@ -128,22 +128,26 @@ export async function generateSnykImportedTargets( listProjects(requestManager, orgId, projectFilters), listIntegrations(requestManager, orgId), ]); - const { projects } = resProjects; - const scmTargets = projects + const { data } = resProjects; + + console.log(data) + + const scmTargets = data .filter((p) => integrationTypes.includes( - p.origin as SupportedIntegrationTypesToListSnykTargets, + p.attributes.origin as SupportedIntegrationTypesToListSnykTargets, ), ) .map((p) => { const target = targetGenerators[ - p.origin as SupportedIntegrationTypesToListSnykTargets - ](p); + p.attributes.origin as SupportedIntegrationTypesToListSnykTargets + ](p.attributes); return { target, - integrationId: resIntegrations[p.origin], + integrationId: resIntegrations[p.attributes.origin], }; }); + console.log("scmTarget: ", scmTargets) const uniqueTargets: Set = new Set(); const orgTargets: Target[] = []; if (!scmTargets.length || scmTargets.length === 0) { diff --git a/test/lib/org.test.ts b/test/lib/org.test.ts index bb3c8193..3146b98f 100644 --- a/test/lib/org.test.ts +++ b/test/lib/org.test.ts @@ -78,7 +78,7 @@ describe('listProjects', () => { }, projects: expect.any(Array), }); - expect(res.projects[0]).toMatchObject({ + expect(res.data[0]).toMatchObject({ name: expect.any(String), branch: expect.any(String), }); diff --git a/test/scripts/generate-imported-targets-from-snyk.test.ts b/test/scripts/generate-imported-targets-from-snyk.test.ts index 3b728337..a46231f4 100644 --- a/test/scripts/generate-imported-targets-from-snyk.test.ts +++ b/test/scripts/generate-imported-targets-from-snyk.test.ts @@ -14,6 +14,7 @@ import { SupportedIntegrationTypesToListSnykTargets } from '../../src/lib/types' const ORG_ID = process.env.TEST_ORG_ID as string; const SNYK_API_TEST = process.env.SNYK_API_TEST as string; +const SNYK_API_V3_TEST = process.env.SNYK_API_V3_TEST as string; const GROUP_ID = process.env.TEST_GROUP_ID as string; jest.unmock('snyk-request-manager'); @@ -23,6 +24,7 @@ describe('Generate imported targets based on Snyk data', () => { let logs: string[]; const OLD_ENV = process.env; process.env.SNYK_API = SNYK_API_TEST; + process.env.SNYK_API_V3 = SNYK_API_V3_TEST; process.env.SNYK_TOKEN = process.env.SNYK_TOKEN_TEST; process.env.SNYK_LOG_PATH = __dirname; @@ -91,7 +93,7 @@ describe('Generate imported targets based on Snyk data', () => { expect(importedTargetsLog).toMatch(targets[0].integrationId); }, 240000); - it('succeeds to generate targets for Org + Azure', async () => { + it.only('succeeds to generate targets for Org + Azure', async () => { const logFiles = generateLogsPaths(__dirname, ORG_ID); logs = Object.values(logFiles); const { @@ -101,6 +103,7 @@ describe('Generate imported targets based on Snyk data', () => { } = await generateSnykImportedTargets({ orgId: ORG_ID }, [ SupportedIntegrationTypesToListSnykTargets.AZURE_REPOS, ]); + console.log("(targets[0]: ", targets[0]) expect(failedOrgs).toEqual([]); expect(fileName).toEqual(path.resolve(__dirname, IMPORT_LOG_NAME)); expect(targets[0]).toMatchObject({ @@ -212,7 +215,7 @@ describe('projectToTarget', () => { it('succeed to convert Github / Gitlab project name to target', async () => { const project = { name: 'lili-snyk/huge-monorepo:cockroach/build/builder/Dockerfile', - branch: 'main', + targetReference: 'main', }; const target = projectToTarget(project); expect(target).toEqual({ @@ -235,7 +238,7 @@ describe('projectToTarget', () => { it('succeed to convert Azure project name to target', async () => { const project = { name: 'Test 105/goof.git:Dockerfile', - branch: 'master', + targetReference: 'master', }; const target = projectToTarget(project); expect(target).toEqual({ @@ -248,7 +251,7 @@ describe('projectToTarget', () => { it('succeed to convert Bitbucket server project name to target', async () => { const project = { name: 'antoine-snyk-demo/TestRepoAntoine:goof/package.json', - branch: 'master', + targetReference: 'master', }; const target = bitbucketServerProjectToTarget(project); expect(target).toEqual({