Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix: pagination
Browse files Browse the repository at this point in the history
mathild3r committed Jul 4, 2022
1 parent 4c7743b commit 1d80c61
Showing 4 changed files with 77 additions and 71 deletions.
1 change: 0 additions & 1 deletion docs/import.md
Original file line number Diff line number Diff line change
@@ -187,7 +187,6 @@ If you have any tests ot fixtures that should be ignored, please set the `exclus
- `SNYK_LOG_PATH` - the path to folder where all logs should be saved,it is recommended creating a dedicated logs folder per import you have running. (Note: all logs will append)
- `CONCURRENT_IMPORTS` (optional) defaults to 15 repos at a time, which is the recommended amount to import at once as a max. Just 1 repo may have many projects inside which can trigger a many files at once to be requested from the user's SCM instance and some may have rate limiting in place. This script aims to help reduce the risk of hitting a rate limit.
- `SNYK_API` (optional) defaults to `https://snyk.io/api/v1`
- `SNYK_API_REST` (optional) defaults to `https://api.snyk.io/rest`

## 3. Download & run

139 changes: 76 additions & 63 deletions src/lib/api/org/index.ts
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@ import * as debugLib from 'debug';
import { getApiToken } from '../../get-api-token';
import { getSnykHost } from '../../get-snyk-host';
import { SnykProject, v3ProjectData } from '../../types';
import { map, StringNullableChain } from 'lodash';
import { StringNullableChain } from 'lodash';

const debug = debugLib('snyk:api-group');

@@ -165,6 +165,7 @@ interface ProjectsFilters {
isMonitored?: boolean; // If set to true, only include projects which are monitored, if set to false, only include projects which are not monitored
}


export async function listProjects(
requestManager: requestsManager,
orgId: string,
@@ -181,87 +182,99 @@ export async function listProjects(
);
}

try {
const res = await requestManager.request({
verb: 'get',
url: `/orgs/${orgId.trim()}/projects?version=2022-06-08~beta`,
body: JSON.stringify(filters),
useRESTApi: true,
});

const statusCode = res.statusCode || res.status;
if (!statusCode || statusCode !== 200) {
throw new Error(
'Expected a 200 response, instead received: ' +
JSON.stringify({ data: res.data, status: statusCode }),
);
}

let v3responseData = res.data as v3ProjectsResponse
let projectData = convertToSnykProject(orgId.trim(), v3responseData)

while (v3responseData.links.next) {
const res = await requestManager.request({
verb: 'get',
url: v3responseData.links.next,
body: JSON.stringify(filters),
useRESTApi: true,
});

const statusCode = res.statusCode || res.status;
if (!statusCode || statusCode !== 200) {
throw new Error(
'Expected a 200 response, instead received: ' +
JSON.stringify({ data: res.data, status: statusCode }),
);
}
v3responseData = res.data as v3ProjectsResponse
const projectDataTmp = projectData
projectData = convertToSnykProject(orgId.trim(), res.data, projectDataTmp)
}

return projectData || {};
} catch (e) {
debug('Failed to update notification settings for ', orgId, e);
throw e;
}
}

export function convertToSnykProject(orgId: string, v3Response: v3ProjectsResponse, v3PreviousDataConverted?: ProjectsResponse): ProjectsResponse {

let snykProjectData = {} as ProjectsResponse

const projects = await listAllProjects(requestManager, orgId, filters)

snykProjectData = {
org: {
name: "",
id: orgId,
},
projects: getProjectData(v3Response.data, v3PreviousDataConverted)
projects: projects
}

return snykProjectData;
}

async function listAllProjects(requestManager: requestsManager,
orgId: string,
filters?: ProjectsFilters,
): Promise<SnykProject[]> {

let lastPage = false;
let projectsList: SnykProject[] = [];
let pageCount = 1;
let nextPageLink: string | undefined = undefined;
while (!lastPage) {
debug(`Fetching page ${pageCount}\n`);
try {
const {
projects,
next,
}: {
projects: SnykProject[];
next?: string;
} = await getProject(requestManager, orgId, filters, nextPageLink);

projectsList = projectsList.concat(projects);
next
? ((lastPage = false), (nextPageLink = next))
: ((lastPage = true), (nextPageLink = ''));
pageCount++;
} catch (e) {
debug('Failed to update notification settings for ', orgId, e);
throw e;
}
}
return projectsList
}

async function getProject(requestManager: requestsManager,
orgId: string,
filters?: ProjectsFilters,
nextPageLink?: string,
): Promise< { projects: SnykProject[], next?: string } > {

return snykProjectData
const url = nextPageLink ? nextPageLink : `/orgs/${orgId.trim()}/projects?version=2022-06-08~beta`
const res = await requestManager.request({
verb: 'get',
url: url,
body: JSON.stringify(filters),
useRESTApi: true,
});

const statusCode = res.statusCode || res.status;
if (!statusCode || statusCode !== 200) {
throw new Error(
'Expected a 200 response, instead received: ' +
JSON.stringify({ data: res.data, status: statusCode }),
);
}

const v3responseData = res.data as v3ProjectsResponse

const projects = convertToSnykProject(v3responseData.data)
const next = v3responseData.links.next

return { projects, next}

}

function getProjectData(projectData: v3ProjectData[], v3PreviousDataConverted?: ProjectsResponse) : SnykProject[] {
function convertToSnykProject(projectData: v3ProjectData[]) : SnykProject[] {

let projects = [] as SnykProject[]
const projectsTmp = [] as SnykProject[]
const projects: SnykProject[] = [];

for (const project of projectData) {
const projectTmp = {} as SnykProject
projectTmp.id = project.id ? project.id : ""
projectTmp.branch = project.attributes.targetReference ? project.attributes.targetReference : null
projectTmp.id = project.id
projectTmp.branch = project.attributes.targetReference
projectTmp.created = project.attributes.created
projectTmp.origin = project.attributes.origin
projectTmp.name = project.attributes.name
projectsTmp.push(projectTmp)
projects.push(projectTmp)
}

if (v3PreviousDataConverted) {
projects = [...v3PreviousDataConverted.projects, ...projectsTmp]
} else {
projects = projectsTmp
}

return projects
}

2 changes: 1 addition & 1 deletion src/lib/types.ts
Original file line number Diff line number Diff line change
@@ -175,7 +175,7 @@ export interface v3ProjectsAttributes {
origin: string;
status: string;
tags: unknown;
targetReference: string | undefined;
targetReference: string | null;
type: string;
}

6 changes: 0 additions & 6 deletions test/system/list:imported.test.ts
Original file line number Diff line number Diff line change
@@ -61,7 +61,6 @@ describe('`snyk-api-import list:imported <...>`', () => {
SNYK_TOKEN: process.env.SNYK_TOKEN_TEST,
SNYK_API: process.env.SNYK_API_TEST,
SNYK_LOG_PATH: __dirname,
SNYK_API_REST: process.env.SNYK_API_REST,
},
},
(err, stdout, stderr) => {
@@ -93,7 +92,6 @@ describe('`snyk-api-import list:imported <...>`', () => {
SNYK_TOKEN: process.env.SNYK_TOKEN_TEST,
SNYK_API: process.env.SNYK_API_TEST,
SNYK_LOG_PATH: __dirname,
SNYK_API_REST: process.env.SNYK_API_REST,
},
},
(err, stdout, stderr) => {
@@ -125,7 +123,6 @@ describe('`snyk-api-import list:imported <...>`', () => {
SNYK_TOKEN: process.env.SNYK_TOKEN_TEST,
SNYK_API: process.env.SNYK_API_TEST,
SNYK_LOG_PATH: __dirname,
SNYK_API_REST: process.env.SNYK_API_REST,
},
},
(err, stdout, stderr) => {
@@ -157,7 +154,6 @@ describe('`snyk-api-import list:imported <...>`', () => {
SNYK_TOKEN: process.env.SNYK_TOKEN_TEST,
SNYK_API: process.env.SNYK_API_TEST,
SNYK_LOG_PATH: __dirname,
SNYK_API_REST: process.env.SNYK_API_REST,
},
},
(err, stdout, stderr) => {
@@ -188,7 +184,6 @@ describe('`snyk-api-import list:imported <...>`', () => {
SNYK_TOKEN: process.env.SNYK_TOKEN_TEST,
SNYK_API: process.env.SNYK_API_TEST,
SNYK_LOG_PATH: __dirname,
SNYK_API_REST: process.env.SNYK_API_REST,
},
},
(err, stdout, stderr) => {
@@ -220,7 +215,6 @@ describe('`snyk-api-import list:imported <...>`', () => {
SNYK_TOKEN: process.env.SNYK_TOKEN_TEST,
SNYK_API: process.env.SNYK_API_TEST,
SNYK_LOG_PATH: __dirname,
SNYK_API_REST: process.env.SNYK_API_REST,
},
},
(err, stdout, stderr) => {

0 comments on commit 1d80c61

Please sign in to comment.