feat: use api v3 get all projects endpoint

docs/import.md

@@ -187,6 +187,7 @@ If you have any tests ot fixtures that should be ignored, please set the `exclus
 - `SNYK_LOG_PATH` - the path to folder where all logs should be saved,it is recommended creating a dedicated logs folder per import you have running. (Note: all logs will append)
 - `CONCURRENT_IMPORTS` (optional) defaults to 15 repos at a time, which is the recommended amount to import at once as a max. Just 1 repo may have many projects inside which can trigger a many files at once to be requested from the user's SCM instance and some may have rate limiting in place. This script aims to help reduce the risk of hitting a rate limit.
 - `SNYK_API` (optional) defaults to `https://snyk.io/api/v1`
+- `SNYK_API_V3` (optional) defaults to `https://api.snyk.io/rest`
 
 ## 3. Download & run
 

src/lib/api/org/index.ts

@@ -3,8 +3,8 @@ import { requestsManager } from 'snyk-request-manager';
 import * as debugLib from 'debug';
 import { getApiToken } from '../../get-api-token';
 import { getSnykHost } from '../../get-snyk-host';
-import { SnykProject } from '../../types';
-import { StringNullableChain } from 'lodash';
+import { SnykProject, v3ProjectData } from '../../types';
+import { map, StringNullableChain } from 'lodash';
 
 const debug = debugLib('snyk:api-group');
 
@@ -142,6 +142,22 @@ interface ProjectsResponse {
   projects: SnykProject[];
 }
 
+
+interface v3ProjectsResponse {
+  data: v3ProjectData[];
+  jsonapi: {
+    version: string;
+  };
+  links: {
+    first: string;
+    last: string;
+    next: string;
+    prev: string;
+    related: string;
+    self: string;
+  };
+}
+
 interface ProjectsFilters {
   name?: StringNullableChain; // If supplied, only projects that have a name that starts with this value will be returned
   origin?: string; //If supplied, only projects that exactly match this origin will be returned
@@ -150,7 +166,6 @@ interface ProjectsFilters {
 }
 
 export async function listProjects(
-  requestManager: requestsManager,
   orgId: string,
   filters?: ProjectsFilters,
 ): Promise<ProjectsResponse> {
@@ -165,10 +180,17 @@ export async function listProjects(
     );
   }
 
+  const oldsnykApiVariable = process.env.SNYK_API
+  process.env.SNYK_API = 'https://api.snyk.io/rest'
+
+  const requestManager = new requestsManager({
+    userAgentPrefix: 'snyk-api-import',
+  });
+
   try {
     const res = await requestManager.request({
       verb: 'post',
-      url: `/org/${orgId.trim()}/projects`,
+      url: `/orgs/${orgId.trim()}/projects?version=2021-06-04`,
       body: JSON.stringify(filters),
     });
 
@@ -179,9 +201,73 @@ export async function listProjects(
           JSON.stringify({ data: res.data, status: statusCode }),
       );
     }
-    return res.data || {};
+
+
+    let v3responseData = res.data as v3ProjectsResponse
+    let projectData = convertToSnykProject(v3responseData)
+
+    while (v3responseData.links.next) {
+      const res = await requestManager.request({
+        verb: 'post',
+        url: v3responseData.links.next,
+        body: JSON.stringify(filters),
+      });
+
+      const statusCode = res.statusCode || res.status;
+      if (!statusCode || statusCode !== 200) {
+        throw new Error(
+          'Expected a 200 response, instead received: ' +
+            JSON.stringify({ data: res.data, status: statusCode }),
+        );
+      }
+      v3responseData = res.data as v3ProjectsResponse
+      projectData = convertToSnykProject(res.data, projectData)
+    }
+
+    process.env.SNYK_API = oldsnykApiVariable
+    return projectData || {};
   } catch (e) {
+    process.env.SNYK_API = oldsnykApiVariable
     debug('Failed to update notification settings for ', orgId, e);
     throw e;
   }
 }
+
+export function convertToSnykProject(v3Response: v3ProjectsResponse, v3PreviousDataConverted?: ProjectsResponse): ProjectsResponse {
+
+  let snykProjectData = {} as ProjectsResponse 
+
+  snykProjectData = {
+    org: {
+      name: "",
+      id: v3Response.data[0].relationships.org.data.id,
+    },
+    projects: getProjectData(v3Response.data, v3PreviousDataConverted)
+  }
+
+  return snykProjectData
+}
+
+function getProjectData(projectData: v3ProjectData[], v3PreviousDataConverted?: ProjectsResponse) : SnykProject[] {
+
+  let projects = [] as SnykProject[]
+  const projectTmp = {} as SnykProject
+
+  if (v3PreviousDataConverted) {
+    projects = v3PreviousDataConverted.projects
+  }
+
+  projectData.forEach( project => {
+
+    projectTmp.id = project.id
+    projectTmp.branch = project.attributes.targetReference ? project.attributes.targetReference : null
+    projectTmp.created = project.attributes.created
+    projectTmp.origin = project.attributes.origin
+    projectTmp.name = project.attributes.name
+    projectTmp.importingUser.id = project.relationships.importingUser.data.id
+
+    projects.push(projectTmp)
+  })
+
+  return projects
+}

src/lib/types.ts

@@ -117,15 +117,15 @@ export interface CommandResult {
 }
 
 export interface SnykProject {
-  name: string;
-  id: string;
-  created: string;
-  origin: string;
-  type: string;
-  readOnly: boolean;
-  testFrequency: string;
-  isMonitored: boolean;
-  totalDependencies: number;
+  name: string; 
+  id: string; 
+  created: string; 
+  origin: string; 
+  type: string; 
+  readOnly: boolean; 
+  testFrequency: string; 
+  isMonitored: boolean; 
+  totalDependencies: number; 
   issueCountsBySeverity: {
     low: number;
     high: number;
@@ -135,17 +135,18 @@ export interface SnykProject {
   remoteRepoUrl: string; // URL of the repo
   lastTestedDate: string;
   browseUrl: string;
-  owner: string | null;
-  importingUser: ImportingUser;
-  tags: unknown[];
+  owner: string | null; 
+  importingUser: ImportingUser; 
+  tags: unknown[]; 
   attributes: {
-    criticality: unknown[];
-    lifecycle: unknown[];
-    environment: unknown[];
+    criticality: unknown[];  
+    lifecycle: unknown[];  
+    environment: unknown[]; 
   };
-  branch: string | null;
+  branch: string | null;  
 }
 
+
 export interface Org {
   name: string;
   id: string;
@@ -156,3 +157,40 @@ export interface Org {
     id: string;
   };
 }
+
+
+export interface v3ProjectData {
+  attributes: v3ProjectsAttributes;
+  id: string;
+  relationships: v3ProjectsRelationships;
+  type: string
+}
+
+export interface v3ProjectsAttributes {
+  businessCriticality: string;
+  created: string;
+  environment: string;
+  lifecycle: string;
+  name: string;
+  origin: string;
+  status: string;
+  tags: unknown;
+  targetReference: string | undefined;
+  type: string;
+}
+
+export interface v3ProjectsRelationships {
+  importingUser: v3ProjectsRelashionshipData;
+  org: v3ProjectsRelashionshipData;
+  owner: v3ProjectsRelashionshipData;
+  target: v3ProjectsRelashionshipData;
+}
+
+export interface v3ProjectsRelashionshipData {
+  data: {
+    id: string;
+    type: string;
+  }
+  links: unknown;
+  meta: unknown;
+}

src/scripts/generate-imported-targets-from-snyk.ts

@@ -125,25 +125,27 @@ export async function generateSnykImportedTargets(
       const { id: orgId, name, slug } = org;
       try {
         const [resProjects, resIntegrations] = await Promise.all([
-          listProjects(requestManager, orgId, projectFilters),
+          listProjects(orgId, projectFilters),
           listIntegrations(requestManager, orgId),
         ]);
+
         const { projects } = resProjects;
         const scmTargets = projects
-          .filter((p) =>
-            integrationTypes.includes(
-              p.origin as SupportedIntegrationTypesToListSnykTargets,
-            ),
-          )
-          .map((p) => {
-            const target = targetGenerators[
-              p.origin as SupportedIntegrationTypesToListSnykTargets
-            ](p);
-            return {
-              target,
-              integrationId: resIntegrations[p.origin],
-            };
-          });
+        .filter((p) =>
+          integrationTypes.includes(
+            p.origin as SupportedIntegrationTypesToListSnykTargets,
+          ),
+        )
+        .map((p) => {
+          const target = targetGenerators[
+            p.origin as SupportedIntegrationTypesToListSnykTargets
+          ](p);
+          return {
+            target,
+            integrationId: resIntegrations[p.origin],
+          };
+        });
+
         const uniqueTargets: Set<string> = new Set();
         const orgTargets: Target[] = [];
         if (!scmTargets.length || scmTargets.length === 0) {

test/scripts/generate-imported-targets-from-snyk.test.ts

@@ -91,7 +91,7 @@ describe('Generate imported targets based on Snyk data', () => {
     expect(importedTargetsLog).toMatch(targets[0].integrationId);
   }, 240000);
 
-  it('succeeds to generate targets for Org + Azure', async () => {
+  it.only('succeeds to generate targets for Org + Azure', async () => {
     const logFiles = generateLogsPaths(__dirname, ORG_ID);
     logs = Object.values(logFiles);
     const {