feat: move api v3 for project endpoint

snyk · Jul 1, 2022 · 4c7743b · 4c7743b
1 parent b92217f
commit 4c7743b
Show file tree

Hide file tree

Showing 7 changed files with 34 additions and 34 deletions.
diff --git a/docs/import.md b/docs/import.md
@@ -187,7 +187,7 @@ If you have any tests ot fixtures that should be ignored, please set the `exclus
 - `SNYK_LOG_PATH` - the path to folder where all logs should be saved,it is recommended creating a dedicated logs folder per import you have running. (Note: all logs will append)
 - `CONCURRENT_IMPORTS` (optional) defaults to 15 repos at a time, which is the recommended amount to import at once as a max. Just 1 repo may have many projects inside which can trigger a many files at once to be requested from the user's SCM instance and some may have rate limiting in place. This script aims to help reduce the risk of hitting a rate limit.
 - `SNYK_API` (optional) defaults to `https://snyk.io/api/v1`
-- `SNYK_API_V3` (optional) defaults to `https://api.snyk.io/rest`
+- `SNYK_API_REST` (optional) defaults to `https://api.snyk.io/rest`
 
 ## 3. Download & run
 

diff --git a/package.json b/package.json
@@ -51,7 +51,7 @@
     "p-map": "4.0.0",
     "parse-link-header": "2.0.0",
     "sleep-promise": "8.0.1",
-    "snyk-request-manager": "1.6.0",
+    "snyk-request-manager": "1.7.1",
     "source-map-support": "^0.5.16",
     "split": "1.0.1",
     "yargs": "16.2.0"

diff --git a/src/lib/api/org/index.ts b/src/lib/api/org/index.ts
@@ -166,6 +166,7 @@ interface ProjectsFilters {
 }
 
 export async function listProjects(
+  requestManager: requestsManager,
   orgId: string,
   filters?: ProjectsFilters,
 ): Promise<ProjectsResponse> {
@@ -180,18 +181,12 @@ export async function listProjects(
     );
   }
 
-  const oldsnykApiVariable = process.env.SNYK_API
-  process.env.SNYK_API = 'https://api.snyk.io/rest'
-
-  const requestManager = new requestsManager({
-    userAgentPrefix: 'snyk-api-import',
-  });
-
   try {
     const res = await requestManager.request({
-      verb: 'post',
-      url: `/orgs/${orgId.trim()}/projects?version=2021-06-04`,
+      verb: 'get',
+      url: `/orgs/${orgId.trim()}/projects?version=2022-06-08~beta`,
       body: JSON.stringify(filters),
+      useRESTApi: true,
     });
 
     const statusCode = res.statusCode || res.status;
@@ -202,15 +197,15 @@ export async function listProjects(
       );
     }
 
-
     let v3responseData = res.data as v3ProjectsResponse
-    let projectData = convertToSnykProject(v3responseData)
+    let projectData = convertToSnykProject(orgId.trim(), v3responseData)
 
     while (v3responseData.links.next) {
       const res = await requestManager.request({
-        verb: 'post',
+        verb: 'get',
         url: v3responseData.links.next,
         body: JSON.stringify(filters),
+        useRESTApi: true,
       });
 
       const statusCode = res.statusCode || res.status;
@@ -221,26 +216,25 @@ export async function listProjects(
         );
       }
       v3responseData = res.data as v3ProjectsResponse
-      projectData = convertToSnykProject(res.data, projectData)
+      const projectDataTmp = projectData
+      projectData = convertToSnykProject(orgId.trim(), res.data, projectDataTmp)
     }
 
-    process.env.SNYK_API = oldsnykApiVariable
     return projectData || {};
   } catch (e) {
-    process.env.SNYK_API = oldsnykApiVariable
     debug('Failed to update notification settings for ', orgId, e);
     throw e;
   }
 }
 
-export function convertToSnykProject(v3Response: v3ProjectsResponse, v3PreviousDataConverted?: ProjectsResponse): ProjectsResponse {
+export function convertToSnykProject(orgId: string, v3Response: v3ProjectsResponse, v3PreviousDataConverted?: ProjectsResponse): ProjectsResponse {
 
   let snykProjectData = {} as ProjectsResponse 
 
   snykProjectData = {
     org: {
       name: "",
-      id: v3Response.data[0].relationships.org.data.id,
+      id: orgId,
     },
     projects: getProjectData(v3Response.data, v3PreviousDataConverted)
   }
@@ -251,23 +245,23 @@ export function convertToSnykProject(v3Response: v3ProjectsResponse, v3PreviousD
 function getProjectData(projectData: v3ProjectData[], v3PreviousDataConverted?: ProjectsResponse) : SnykProject[] {
 
   let projects = [] as SnykProject[]
-  const projectTmp = {} as SnykProject
-
-  if (v3PreviousDataConverted) {
-    projects = v3PreviousDataConverted.projects
-  }
-
-  projectData.forEach( project => {
+  const projectsTmp = [] as SnykProject[]
 
-    projectTmp.id = project.id
+  for (const project of projectData) {  
+    const projectTmp = {} as SnykProject
+    projectTmp.id = project.id ? project.id : ""
     projectTmp.branch = project.attributes.targetReference ? project.attributes.targetReference : null
     projectTmp.created = project.attributes.created
     projectTmp.origin = project.attributes.origin
     projectTmp.name = project.attributes.name
-    projectTmp.importingUser.id = project.relationships.importingUser.data.id
-
-    projects.push(projectTmp)
-  })
+    projectsTmp.push(projectTmp)
+  }
 
+  if (v3PreviousDataConverted) {
+    projects = [...v3PreviousDataConverted.projects, ...projectsTmp]
+  } else {
+    projects = projectsTmp
+  }
+
   return projects
 }
diff --git a/src/scripts/generate-imported-targets-from-snyk.ts b/src/scripts/generate-imported-targets-from-snyk.ts
@@ -125,10 +125,9 @@ export async function generateSnykImportedTargets(
       const { id: orgId, name, slug } = org;
       try {
         const [resProjects, resIntegrations] = await Promise.all([
-          listProjects(orgId, projectFilters),
+          listProjects(requestManager, orgId, projectFilters),
           listIntegrations(requestManager, orgId),
         ]);
-
         const { projects } = resProjects;
         const scmTargets = projects
         .filter((p) =>

diff --git a/test/scripts/generate-imported-targets-from-snyk.test.ts b/test/scripts/generate-imported-targets-from-snyk.test.ts
@@ -91,7 +91,7 @@ describe('Generate imported targets based on Snyk data', () => {
     expect(importedTargetsLog).toMatch(targets[0].integrationId);
   }, 240000);
 
-  it.only('succeeds to generate targets for Org + Azure', async () => {
+  it('succeeds to generate targets for Org + Azure', async () => {
     const logFiles = generateLogsPaths(__dirname, ORG_ID);
     logs = Object.values(logFiles);
     const {

diff --git a/test/scripts/github-enterprise-import-targets.json b/test/scripts/github-enterprise-import-targets.json
@@ -0,0 +1 @@
+{"targets":[{"target":{"fork":false,"name":"goof","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"shallow-goof","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"ruby-app-rails-3","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"maven-simple-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"fixture-maven-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"goof-readers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"goof-writers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"goof-admins","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple-readers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple-writers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple-admins","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"mono-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"python-pip-app-with-vulns","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"python-pip3-app-no-policy","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"python-pip3-app","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"nested-manifest-files-only","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"goof-private","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"dotnet-project-json","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"fixture-big-file","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"dotnet-multiple-frameworks","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"js-multi-level-mono-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"js-pull-requests","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-missing-package-prop","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-no-proj-name","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-not-vuln-project","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-vuln-project","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"php-composer-lockfiles","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"},{"target":{"fork":false,"name":"docker-goof","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-********-********","orgId":"org-id"}]}
diff --git a/test/system/list:imported.test.ts b/test/system/list:imported.test.ts
@@ -61,6 +61,7 @@ describe('`snyk-api-import list:imported <...>`', () => {
           SNYK_TOKEN: process.env.SNYK_TOKEN_TEST,
           SNYK_API: process.env.SNYK_API_TEST,
           SNYK_LOG_PATH: __dirname,
+          SNYK_API_REST: process.env.SNYK_API_REST,
         },
       },
       (err, stdout, stderr) => {
@@ -92,6 +93,7 @@ describe('`snyk-api-import list:imported <...>`', () => {
           SNYK_TOKEN: process.env.SNYK_TOKEN_TEST,
           SNYK_API: process.env.SNYK_API_TEST,
           SNYK_LOG_PATH: __dirname,
+          SNYK_API_REST: process.env.SNYK_API_REST,
         },
       },
       (err, stdout, stderr) => {
@@ -123,6 +125,7 @@ describe('`snyk-api-import list:imported <...>`', () => {
           SNYK_TOKEN: process.env.SNYK_TOKEN_TEST,
           SNYK_API: process.env.SNYK_API_TEST,
           SNYK_LOG_PATH: __dirname,
+          SNYK_API_REST: process.env.SNYK_API_REST,
         },
       },
       (err, stdout, stderr) => {
@@ -154,6 +157,7 @@ describe('`snyk-api-import list:imported <...>`', () => {
           SNYK_TOKEN: process.env.SNYK_TOKEN_TEST,
           SNYK_API: process.env.SNYK_API_TEST,
           SNYK_LOG_PATH: __dirname,
+          SNYK_API_REST: process.env.SNYK_API_REST,
         },
       },
       (err, stdout, stderr) => {
@@ -184,6 +188,7 @@ describe('`snyk-api-import list:imported <...>`', () => {
           SNYK_TOKEN: process.env.SNYK_TOKEN_TEST,
           SNYK_API: process.env.SNYK_API_TEST,
           SNYK_LOG_PATH: __dirname,
+          SNYK_API_REST: process.env.SNYK_API_REST,
         },
       },
       (err, stdout, stderr) => {
@@ -215,6 +220,7 @@ describe('`snyk-api-import list:imported <...>`', () => {
           SNYK_TOKEN: process.env.SNYK_TOKEN_TEST,
           SNYK_API: process.env.SNYK_API_TEST,
           SNYK_LOG_PATH: __dirname,
+          SNYK_API_REST: process.env.SNYK_API_REST,
         },
       },
       (err, stdout, stderr) => {
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"targets":[{"target":{"fork":false,"name":"goof","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-******-****","orgId":"org-id"},{"target":{"fork":false,"name":"shallow-goof","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"ruby-app-rails-3","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"maven-simple-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"fixture-maven-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"goof-readers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"goof-writers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"goof-admins","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple-readers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple-writers","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"monorepo-simple-admins","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"mono-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"python-pip-app-with-vulns","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"python-pip3-app-no-policy","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"python-pip3-app","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"nested-manifest-files-only","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"goof-private","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"dotnet-project-json","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"fixture-big-file","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"dotnet-multiple-frameworks","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"js-multi-level-mono-repo","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"js-pull-requests","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-missing-package-prop","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-no-proj-name","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-not-vuln-project","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"composer-mono-vuln-project","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"php-composer-lockfiles","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-****","orgId":"org-id"},{"target":{"fork":false,"name":"docker-goof","owner":"snyk-fixtures","branch":"master"},"integrationId":"github-enterprise-****-******","orgId":"org-id"}]}