feat: use api v3 get all projects endpoint

docs/import.md

@@ -187,6 +187,7 @@ If you have any tests ot fixtures that should be ignored, please set the `exclus
 - `SNYK_LOG_PATH` - the path to folder where all logs should be saved,it is recommended creating a dedicated logs folder per import you have running. (Note: all logs will append)
 - `CONCURRENT_IMPORTS` (optional) defaults to 15 repos at a time, which is the recommended amount to import at once as a max. Just 1 repo may have many projects inside which can trigger a many files at once to be requested from the user's SCM instance and some may have rate limiting in place. This script aims to help reduce the risk of hitting a rate limit.
 - `SNYK_API` (optional) defaults to `https://snyk.io/api/v1`
+- `SNYK_API_V3` (optional) defaults to `https://api.snyk.io/rest`
 
 ## 3. Download & run
 

src/lib/api/org/index.ts

@@ -3,7 +3,7 @@ import { requestsManager } from 'snyk-request-manager';
 import * as debugLib from 'debug';
 import { getApiToken } from '../../get-api-token';
 import { getSnykHost } from '../../get-snyk-host';
-import { SnykProject } from '../../types';
+import { SnykProject, v3ProjectData } from '../../types';
 import { StringNullableChain } from 'lodash';
 
 const debug = debugLib('snyk:api-group');
@@ -142,6 +142,15 @@ interface ProjectsResponse {
   projects: SnykProject[];
 }
 
+
+interface v3ProjectsResponse {
+  data: v3ProjectData[];
+  jsonapi: {
+    version: string;
+  };
+  links: unknown;
+}
+
 interface ProjectsFilters {
   name?: StringNullableChain; // If supplied, only projects that have a name that starts with this value will be returned
   origin?: string; //If supplied, only projects that exactly match this origin will be returned
@@ -153,9 +162,9 @@ export async function listProjects(
   requestManager: requestsManager,
   orgId: string,
   filters?: ProjectsFilters,
-): Promise<ProjectsResponse> {
+): Promise<v3ProjectsResponse> {
   getApiToken();
-  getSnykHost();
+  const SNYK_API = getSnykHost("v3");
   debug(`Listing all projects for org: ${orgId}`);
 
   if (!orgId) {
@@ -168,7 +177,7 @@ export async function listProjects(
   try {
     const res = await requestManager.request({
       verb: 'post',
-      url: `/org/${orgId.trim()}/projects`,
+      url: `${SNYK_API}/orgs/${orgId.trim()}/projects?version=2021-06-04`,
       body: JSON.stringify(filters),
     });
 

src/lib/get-snyk-host.ts

@@ -1,3 +1,8 @@
-export function getSnykHost(): string {
-  return process.env.SNYK_API || 'https://snyk.io/api/v1';
+export function getSnykHost(ApiVersion?: string): string {
+  let apiUrl = 'https://snyk.io/api/v1'
+  if (ApiVersion === "v3") {
+    apiUrl = 'https://api.snyk.io/rest'
+    return process.env.SNYK_API_V3 || apiUrl;
+  }
+  return process.env.SNYK_API || apiUrl;
 }

src/lib/types.ts

@@ -117,15 +117,15 @@ export interface CommandResult {
 }
 
 export interface SnykProject {
-  name: string;
-  id: string;
-  created: string;
-  origin: string;
-  type: string;
-  readOnly: boolean;
-  testFrequency: string;
-  isMonitored: boolean;
-  totalDependencies: number;
+  name: string; 
+  id: string; 
+  created: string; 
+  origin: string; 
+  type: string; 
+  readOnly: boolean; 
+  testFrequency: string; 
+  isMonitored: boolean; 
+  totalDependencies: number; 
   issueCountsBySeverity: {
     low: number;
     high: number;
@@ -135,17 +135,18 @@ export interface SnykProject {
   remoteRepoUrl: string; // URL of the repo
   lastTestedDate: string;
   browseUrl: string;
-  owner: string | null;
-  importingUser: ImportingUser;
-  tags: unknown[];
+  owner: string | null; 
+  importingUser: ImportingUser; 
+  tags: unknown[]; 
   attributes: {
-    criticality: unknown[];
-    lifecycle: unknown[];
-    environment: unknown[];
+    criticality: unknown[];  
+    lifecycle: unknown[];  
+    environment: unknown[]; 
   };
-  branch: string | null;
+  branch: string | null;  
 }
 
+
 export interface Org {
   name: string;
   id: string;
@@ -156,3 +157,40 @@ export interface Org {
     id: string;
   };
 }
+
+
+export interface v3ProjectData {
+  attributes: v3ProjectsAttributes;
+  id: string;
+  relationships: v3ProjectsRelationships;
+  type: string
+}
+
+export interface v3ProjectsAttributes {
+  businessCriticality: string;
+  created: string;
+  environment: string;
+  lifecycle: string;
+  name: string;
+  origin: string;
+  status: string;
+  tags: unknown;
+  targetReference: string | undefined;
+  type: string;
+}
+
+export interface v3ProjectsRelationships {
+  importingUser: v3ProjectsRelashionshipData;
+  org: v3ProjectsRelashionshipData;
+  owner: v3ProjectsRelashionshipData;
+  target: v3ProjectsRelashionshipData;
+}
+
+export interface v3ProjectsRelashionshipData {
+  data: {
+    id: string;
+    type: string;
+  }
+  links: unknown;
+  meta: unknown;
+}

src/scripts/generate-imported-targets-from-snyk.ts

@@ -6,9 +6,9 @@ import * as _ from 'lodash';
 
 import {
   FilePath,
-  SnykProject,
   SupportedIntegrationTypesToListSnykTargets,
   Target,
+  v3ProjectsAttributes,
 } from '../lib/types';
 import {
   getAllOrgs,
@@ -31,42 +31,42 @@ export interface ImportTarget {
 }
 
 export function projectToTarget(
-  project: Pick<SnykProject, 'name' | 'branch'>,
+  projectAttributes: Pick<v3ProjectsAttributes, 'name' | 'targetReference'>,
 ): Target {
-  const [owner, name] = project.name.split(':')[0].split('/');
+  const [owner, name] = projectAttributes.name.split(':')[0].split('/');
   return {
     owner,
-    branch: project.branch || undefined, // TODO: make it not optional
+    branch: projectAttributes.targetReference || undefined, // TODO: make it not optional
     name,
   };
 }
 export function bitbucketServerProjectToTarget(
-  project: Pick<SnykProject, 'name' | 'branch'>,
+  projectAttributes: Pick<v3ProjectsAttributes, 'name' | 'targetReference'>,
 ): Target {
-  const [projectKey, repoSlug] = project.name.split(':')[0].split('/');
+  const [projectKey, repoSlug] = projectAttributes.name.split(':')[0].split('/');
   return {
     projectKey,
     repoSlug,
   };
 }
 
 export function gitlabProjectToImportLogTarget(
-  project: Pick<SnykProject, 'name' | 'branch'>,
+  projectAttributes: Pick<v3ProjectsAttributes, 'name' | 'targetReference'>,
 ): Target {
   // Gitlab target is only `id` & branch and the Snyk API does not return the id.
   // However we are already logging `name` which for Gitlab is "owner/repo", branch & id so if we use the same name we can match on it
-  const name = project.name.split(':')[0];
+  const name = projectAttributes.name.split(':')[0];
   return {
-    branch: project.branch || undefined, // TODO: make it not optional
+    branch: projectAttributes.targetReference || undefined, // TODO: make it not optional
     name,
   };
 }
 
 export function imageProjectToTarget(
-  project: Pick<SnykProject, 'name'>,
+  projectAttributes: Pick<v3ProjectsAttributes, 'name'>,
 ): Target {
   return {
-    name: project.name,
+    name: projectAttributes.name,
   };
 }
 
@@ -128,22 +128,26 @@ export async function generateSnykImportedTargets(
           listProjects(requestManager, orgId, projectFilters),
           listIntegrations(requestManager, orgId),
         ]);
-        const { projects } = resProjects;
-        const scmTargets = projects
+        const { data } = resProjects;
+
+        console.log(data)
+
+        const scmTargets = data
           .filter((p) =>
             integrationTypes.includes(
-              p.origin as SupportedIntegrationTypesToListSnykTargets,
+              p.attributes.origin as SupportedIntegrationTypesToListSnykTargets,
             ),
           )
           .map((p) => {
             const target = targetGenerators[
-              p.origin as SupportedIntegrationTypesToListSnykTargets
-            ](p);
+              p.attributes.origin as SupportedIntegrationTypesToListSnykTargets
+            ](p.attributes);
             return {
               target,
-              integrationId: resIntegrations[p.origin],
+              integrationId: resIntegrations[p.attributes.origin],
             };
           });
+        console.log("scmTarget: ", scmTargets)
         const uniqueTargets: Set<string> = new Set();
         const orgTargets: Target[] = [];
         if (!scmTargets.length || scmTargets.length === 0) {

test/lib/org.test.ts

@@ -78,7 +78,7 @@ describe('listProjects', () => {
       },
       projects: expect.any(Array),
     });
-    expect(res.projects[0]).toMatchObject({
+    expect(res.data[0]).toMatchObject({
       name: expect.any(String),
       branch: expect.any(String),
     });

test/scripts/generate-imported-targets-from-snyk.test.ts

@@ -14,6 +14,7 @@ import { SupportedIntegrationTypesToListSnykTargets } from '../../src/lib/types'
 
 const ORG_ID = process.env.TEST_ORG_ID as string;
 const SNYK_API_TEST = process.env.SNYK_API_TEST as string;
+const SNYK_API_V3_TEST = process.env.SNYK_API_V3_TEST as string;
 const GROUP_ID = process.env.TEST_GROUP_ID as string;
 
 jest.unmock('snyk-request-manager');
@@ -23,6 +24,7 @@ describe('Generate imported targets based on Snyk data', () => {
   let logs: string[];
   const OLD_ENV = process.env;
   process.env.SNYK_API = SNYK_API_TEST;
+  process.env.SNYK_API_V3 = SNYK_API_V3_TEST;
   process.env.SNYK_TOKEN = process.env.SNYK_TOKEN_TEST;
   process.env.SNYK_LOG_PATH = __dirname;
 
@@ -91,7 +93,7 @@ describe('Generate imported targets based on Snyk data', () => {
     expect(importedTargetsLog).toMatch(targets[0].integrationId);
   }, 240000);
 
-  it('succeeds to generate targets for Org + Azure', async () => {
+  it.only('succeeds to generate targets for Org + Azure', async () => {
     const logFiles = generateLogsPaths(__dirname, ORG_ID);
     logs = Object.values(logFiles);
     const {
@@ -101,6 +103,7 @@ describe('Generate imported targets based on Snyk data', () => {
     } = await generateSnykImportedTargets({ orgId: ORG_ID }, [
       SupportedIntegrationTypesToListSnykTargets.AZURE_REPOS,
     ]);
+    console.log("(targets[0]: ", targets[0])
     expect(failedOrgs).toEqual([]);
     expect(fileName).toEqual(path.resolve(__dirname, IMPORT_LOG_NAME));
     expect(targets[0]).toMatchObject({
@@ -212,7 +215,7 @@ describe('projectToTarget', () => {
   it('succeed to convert Github / Gitlab project name to target', async () => {
     const project = {
       name: 'lili-snyk/huge-monorepo:cockroach/build/builder/Dockerfile',
-      branch: 'main',
+      targetReference: 'main',
     };
     const target = projectToTarget(project);
     expect(target).toEqual({
@@ -235,7 +238,7 @@ describe('projectToTarget', () => {
   it('succeed to convert Azure project name to target', async () => {
     const project = {
       name: 'Test 105/goof.git:Dockerfile',
-      branch: 'master',
+      targetReference: 'master',
     };
     const target = projectToTarget(project);
     expect(target).toEqual({
@@ -248,7 +251,7 @@ describe('projectToTarget', () => {
   it('succeed to convert Bitbucket server project name to target', async () => {
     const project = {
       name: 'antoine-snyk-demo/TestRepoAntoine:goof/package.json',
-      branch: 'master',
+      targetReference: 'master',
     };
     const target = bitbucketServerProjectToTarget(project);
     expect(target).toEqual({