feat: use api v3 get all projects endpoint

package.json

@@ -51,7 +51,7 @@
     "p-map": "4.0.0",
     "parse-link-header": "2.0.0",
     "sleep-promise": "8.0.1",
-    "snyk-request-manager": "1.6.0",
+    "snyk-request-manager": "1.7.1",
     "source-map-support": "^0.5.16",
     "split": "1.0.1",
     "yargs": "16.2.0"

src/lib/api/org/index.ts

@@ -3,7 +3,7 @@ import { requestsManager } from 'snyk-request-manager';
 import * as debugLib from 'debug';
 import { getApiToken } from '../../get-api-token';
 import { getSnykHost } from '../../get-snyk-host';
-import { SnykProject } from '../../types';
+import { SnykProject, v3ProjectData } from '../../types';
 import { StringNullableChain } from 'lodash';
 
 const debug = debugLib('snyk:api-group');
@@ -142,13 +142,30 @@ interface ProjectsResponse {
   projects: SnykProject[];
 }
 
+
+interface v3ProjectsResponse {
+  data: v3ProjectData[];
+  jsonapi: {
+    version: string;
+  };
+  links: {
+    first: string;
+    last: string;
+    next: string;
+    prev: string;
+    related: string;
+    self: string;
+  };
+}
+
 interface ProjectsFilters {
   name?: StringNullableChain; // If supplied, only projects that have a name that starts with this value will be returned
   origin?: string; //If supplied, only projects that exactly match this origin will be returned
   type?: string; //If supplied, only projects that exactly match this type will be returned
   isMonitored?: boolean; // If set to true, only include projects which are monitored, if set to false, only include projects which are not monitored
 }
 
+
 export async function listProjects(
   requestManager: requestsManager,
   orgId: string,
@@ -165,23 +182,99 @@ export async function listProjects(
     );
   }
 
-  try {
-    const res = await requestManager.request({
-      verb: 'post',
-      url: `/org/${orgId.trim()}/projects`,
-      body: JSON.stringify(filters),
-    });
+  let snykProjectData = {} as ProjectsResponse 
 
-    const statusCode = res.statusCode || res.status;
-    if (!statusCode || statusCode !== 200) {
-      throw new Error(
-        'Expected a 200 response, instead received: ' +
-          JSON.stringify({ data: res.data, status: statusCode }),
-      );
+  const projects = await listAllProjects(requestManager, orgId, filters)
+
+  snykProjectData = {
+    org: {
+      name: "",
+      id: orgId,
+    },
+    projects: projects
+  }
+
+  return snykProjectData;
+}
+
+async function listAllProjects(requestManager: requestsManager,
+  orgId: string,
+  filters?: ProjectsFilters,
+  ): Promise<SnykProject[]> {
+
+    let lastPage = false;
+    let projectsList: SnykProject[] = [];
+    let pageCount = 1;
+    let nextPageLink: string | undefined = undefined;
+    while (!lastPage) {
+      debug(`Fetching page ${pageCount}\n`);
+      try {
+        const {
+          projects,
+          next,
+        }: {
+          projects: SnykProject[];
+          next?: string;
+        } = await getProject(requestManager, orgId, filters, nextPageLink);
+
+        projectsList = projectsList.concat(projects);
+        next
+          ? ((lastPage = false), (nextPageLink = next))
+          : ((lastPage = true), (nextPageLink = ''));
+        pageCount++;
+      } catch (e) {
+        debug('Failed to update notification settings for ', orgId, e);
+        throw e;
     }
-    return res.data || {};
-  } catch (e) {
-    debug('Failed to update notification settings for ', orgId, e);
-    throw e;
   }
+  return projectsList
 }
+
+async function getProject(requestManager: requestsManager,
+  orgId: string,
+  filters?: ProjectsFilters,
+  nextPageLink?: string,
+  ): Promise< { projects: SnykProject[], next?: string } > {
+
+    const url = nextPageLink ? nextPageLink : `/orgs/${orgId.trim()}/projects?version=2022-06-08~beta`
+    const res = await requestManager.request({
+    verb: 'get',
+    url: url,
+    body: JSON.stringify(filters),
+    useRESTApi: true,
+  });
+
+  const statusCode = res.statusCode || res.status;
+  if (!statusCode || statusCode !== 200) {
+    throw new Error(
+      'Expected a 200 response, instead received: ' +
+        JSON.stringify({ data: res.data, status: statusCode }),
+    );
+  }
+
+  const v3responseData = res.data as v3ProjectsResponse
+
+  const projects = convertToSnykProject(v3responseData.data)
+  const next =  v3responseData.links.next
+
+  return { projects, next}
+
+}
+
+function convertToSnykProject(projectData: v3ProjectData[]) : SnykProject[] {
+
+  const projects: SnykProject[] = [];
+
+  for (const project of projectData) {  
+    const projectTmp = {} as SnykProject
+    projectTmp.id = project.id 
+    projectTmp.branch = project.attributes.targetReference 
+    projectTmp.created = project.attributes.created
+    projectTmp.origin = project.attributes.origin
+    projectTmp.name = project.attributes.name
+    projects.push(projectTmp)
+  }
+
+  return projects
+}
+

src/lib/types.ts

@@ -117,15 +117,15 @@ export interface CommandResult {
 }
 
 export interface SnykProject {
-  name: string;
-  id: string;
-  created: string;
-  origin: string;
-  type: string;
-  readOnly: boolean;
-  testFrequency: string;
-  isMonitored: boolean;
-  totalDependencies: number;
+  name: string; 
+  id: string; 
+  created: string; 
+  origin: string; 
+  type: string; 
+  readOnly: boolean; 
+  testFrequency: string; 
+  isMonitored: boolean; 
+  totalDependencies: number; 
   issueCountsBySeverity: {
     low: number;
     high: number;
@@ -135,17 +135,18 @@ export interface SnykProject {
   remoteRepoUrl: string; // URL of the repo
   lastTestedDate: string;
   browseUrl: string;
-  owner: string | null;
-  importingUser: ImportingUser;
-  tags: unknown[];
+  owner: string | null; 
+  importingUser: ImportingUser; 
+  tags: unknown[]; 
   attributes: {
-    criticality: unknown[];
-    lifecycle: unknown[];
-    environment: unknown[];
+    criticality: unknown[];  
+    lifecycle: unknown[];  
+    environment: unknown[]; 
   };
-  branch: string | null;
+  branch: string | null;  
 }
 
+
 export interface Org {
   name: string;
   id: string;
@@ -156,3 +157,40 @@ export interface Org {
     id: string;
   };
 }
+
+
+export interface v3ProjectData {
+  attributes: v3ProjectsAttributes;
+  id: string;
+  relationships: v3ProjectsRelationships;
+  type: string
+}
+
+export interface v3ProjectsAttributes {
+  businessCriticality: string;
+  created: string;
+  environment: string;
+  lifecycle: string;
+  name: string;
+  origin: string;
+  status: string;
+  tags: unknown;
+  targetReference: string | null;
+  type: string;
+}
+
+export interface v3ProjectsRelationships {
+  importingUser: v3ProjectsRelashionshipData;
+  org: v3ProjectsRelashionshipData;
+  owner: v3ProjectsRelashionshipData;
+  target: v3ProjectsRelashionshipData;
+}
+
+export interface v3ProjectsRelashionshipData {
+  data: {
+    id: string;
+    type: string;
+  }
+  links: unknown;
+  meta: unknown;
+}

src/scripts/generate-imported-targets-from-snyk.ts

@@ -130,20 +130,21 @@ export async function generateSnykImportedTargets(
         ]);
         const { projects } = resProjects;
         const scmTargets = projects
-          .filter((p) =>
-            integrationTypes.includes(
-              p.origin as SupportedIntegrationTypesToListSnykTargets,
-            ),
-          )
-          .map((p) => {
-            const target = targetGenerators[
-              p.origin as SupportedIntegrationTypesToListSnykTargets
-            ](p);
-            return {
-              target,
-              integrationId: resIntegrations[p.origin],
-            };
-          });
+        .filter((p) =>
+          integrationTypes.includes(
+            p.origin as SupportedIntegrationTypesToListSnykTargets,
+          ),
+        )
+        .map((p) => {
+          const target = targetGenerators[
+            p.origin as SupportedIntegrationTypesToListSnykTargets
+          ](p);
+          return {
+            target,
+            integrationId: resIntegrations[p.origin],
+          };
+        });
+
         const uniqueTargets: Set<string> = new Set();
         const orgTargets: Target[] = [];
         if (!scmTargets.length || scmTargets.length === 0) {

test/scripts/github-enterprise-import-targets.json

@@ -0,0 +1 @@
+{"targets":[{"target":{"fork":false,"name":"goof","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"shallow-goof","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"ruby-app-rails-3","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"maven-simple-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"fixture-maven-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"goof-readers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"goof-writers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"goof-admins","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple-readers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple-writers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple-admins","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"mono-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"python-pip-app-with-vulns","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"python-pip3-app-no-policy","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"python-pip3-app","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"nested-manifest-files-only","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"goof-private","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"dotnet-project-json","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"fixture-big-file","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"dotnet-multiple-frameworks","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"js-multi-level-mono-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"js-pull-requests","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-missing-package-prop","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-no-proj-name","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-not-vuln-project","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-vuln-project","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"php-composer-lockfiles","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"docker-goof","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"}]}