diff --git a/src/cmds/orgs:create.ts b/src/cmds/orgs:create.ts index 786802bf..9b02cc83 100644 --- a/src/cmds/orgs:create.ts +++ b/src/cmds/orgs:create.ts @@ -6,7 +6,7 @@ import { createOrgs } from '../scripts/create-orgs'; export const command = ['orgs:create']; export const desc = - 'Create the Orgs in Snyk based on data file generated with `orgs:data` command'; + 'Create the organizations in Snyk based on data file generated with `orgs:data` command. Output generates key data for created and existing organizations for use to generate project import data.'; export const builder = { file: { required: true, @@ -18,17 +18,26 @@ export const builder = { desc: 'Skip creating an organization if the given name is already taken within the Group.', }, + includeExistingOrgsInOutput: { + required: false, + default: true, + desc: 'Log existing organization information as well as newly created', + }, }; export async function handler(argv: { file: string; + includeExistingOrgsInOutput: boolean; noDuplicateNames?: boolean; }): Promise { try { getLoggingPath(); - const { file, noDuplicateNames } = argv; + const { file, noDuplicateNames, includeExistingOrgsInOutput } = argv; debug('ℹ️ Options: ' + JSON.stringify(argv)); - const res = await createOrgs(file, noDuplicateNames); + const res = await createOrgs(file, { + noDuplicateNames, + includeExistingOrgsInOutput, + }); const orgsMessage = res.orgs.length > 0 diff --git a/src/lib/api/org/index.ts b/src/lib/api/org/index.ts index 22c6bd75..86f5cecc 100644 --- a/src/lib/api/org/index.ts +++ b/src/lib/api/org/index.ts @@ -69,12 +69,12 @@ interface NotificationSettings { export async function setNotificationPreferences( requestManager: requestsManager, orgId: string, - orgData: CreateOrgData, + orgName: string, settings: NotificationSettings = defaultDisabledSettings, ): Promise { getApiToken(); getSnykHost(); - debug(`Disabling notifications for org: ${orgData.name} (${orgId})`); + debug(`Disabling notifications for org: ${orgName} (${orgId})`); if (!orgId) { throw new Error( diff --git a/src/lib/filter-out-existing-orgs.ts b/src/lib/filter-out-existing-orgs.ts index 6d160195..b7a76d0e 100644 --- a/src/lib/filter-out-existing-orgs.ts +++ b/src/lib/filter-out-existing-orgs.ts @@ -1,13 +1,15 @@ +import * as _ from 'lodash'; + import { requestsManager } from 'snyk-request-manager'; import { getAllOrgs } from './get-all-orgs-for-group'; -import { CreateOrgData } from './types'; +import { CreateOrgData, Org } from './types'; export async function filterOutExistingOrgs( requestManager: requestsManager, orgs: CreateOrgData[] = [], groupId: string, ): Promise<{ - existingOrgs: CreateOrgData[]; + existingOrgs: Org[]; newOrgs: CreateOrgData[]; }> { if (!groupId) { @@ -17,17 +19,25 @@ export async function filterOutExistingOrgs( return { existingOrgs: [], newOrgs: [] }; } - const existingOrgs: CreateOrgData[] = []; + const existingOrgs: Org[] = []; const newOrgs: CreateOrgData[] = []; const groupOrgs = await getAllOrgs(requestManager, groupId); const uniqueOrgNames: Set = new Set(groupOrgs.map((org) => org.name)); for (const org of orgs) { - if (uniqueOrgNames.has(org.name)) { - existingOrgs.push(org); - continue; + if (!uniqueOrgNames.has(org.name)) { + newOrgs.push(org); } - newOrgs.push(org); + } + if (existingOrgs.length > 0) { + console.log( + `Skipped creating ${ + existingOrgs.length + } organization(s) as the names were already used in the Group ${groupId}. Organizations skipped: ${existingOrgs + .map((o) => o.name) + .join(', ')}`, + ); } - return { existingOrgs, newOrgs }; + return { existingOrgs: groupOrgs, newOrgs }; } + diff --git a/src/scripts/create-orgs.ts b/src/scripts/create-orgs.ts index 2c9649cc..4cd671f1 100644 --- a/src/scripts/create-orgs.ts +++ b/src/scripts/create-orgs.ts @@ -6,14 +6,14 @@ import { CreatedOrgResponse, createOrg, filterOutExistingOrgs } from '../lib'; import { getLoggingPath } from './../lib'; import { listIntegrations, setNotificationPreferences } from '../lib/api/org'; import { requestsManager } from 'snyk-request-manager'; -import { CreateOrgData } from '../lib/types'; +import { CreateOrgData, Org } from '../lib/types'; import { logCreatedOrg } from '../loggers/log-created-org'; import { writeFile } from '../write-file'; import { FAILED_ORG_LOG_NAME } from '../common'; import { logFailedOrg } from '../loggers/log-failed-org'; const debug = debugLib('snyk:create-orgs-script'); -interface CreatedOrg extends CreatedOrgResponse { +interface NewOrExistingOrg extends CreatedOrgResponse { integrations: { [name: string]: string; }; @@ -23,22 +23,102 @@ interface CreatedOrg extends CreatedOrgResponse { sourceOrgId?: string; } -async function saveCreatedOrgData(orgData: CreatedOrg[]): Promise { +async function saveCreatedOrgData( + orgData: Partial[], +): Promise { const fileName = 'snyk-created-orgs.json'; await writeFile(fileName, ({ orgData } as unknown) as JSON); return fileName; } +async function createNewOrgs( + loggingPath: string, + requestManager: requestsManager, + groupId: string, + orgsToCreate: CreateOrgData[], +): Promise<{ failed: CreateOrgData[]; created: NewOrExistingOrg[] }> { + const failed: CreateOrgData[] = []; + const created: NewOrExistingOrg[] = []; + + for (const orgData of orgsToCreate) { + const { name, sourceOrgId } = orgData; + try { + const org = await createOrg(requestManager, groupId, name, sourceOrgId); + const integrations = + (await listIntegrations(requestManager, org.id)) || {}; + await setNotificationPreferences(requestManager, org.id, org.name); + created.push({ + ...org, + orgId: org.id, + integrations, + groupId, + origName: name, + sourceOrgId, + }); + logCreatedOrg(groupId, name, org, integrations, loggingPath); + } catch (e) { + failed.push({ groupId, name, sourceOrgId }); + const errorMessage = e.data ? e.data.message : e.message; + logFailedOrg( + groupId, + name, + errorMessage || 'Failed to create org, please try again in DEBUG mode.', + ); + debug( + `Failed to create organization with data: ${JSON.stringify(orgData)}`, + e, + ); + } + } + + return { failed, created }; +} + +async function listExistingOrgsData( + requestManager: requestsManager, + existingOrgs: Org[], +): Promise<{ existing: Partial[] }> { + const previouslyCreated: Partial[] = []; + + for (const orgData of existingOrgs) { + const { name, id, group } = orgData; + try { + const integrations = (await listIntegrations(requestManager, id)) || {}; + previouslyCreated.push({ + ...orgData, + name, + orgId: id, + integrations, + groupId: group.id, + origName: name, + }); + } catch (e) { + debug( + `Failed to list integrations for Org: ${orgData.name} (${orgData.id})`, + e, + ); + } + } + return { existing: previouslyCreated }; +} export async function createOrgs( filePath: string, - skipIfOrgNameExists = false, + options: { + noDuplicateNames?: boolean; + includeExistingOrgsInOutput: boolean; + } = { + noDuplicateNames: false, + includeExistingOrgsInOutput: true, + }, loggingPath = getLoggingPath(), ): Promise<{ - orgs: CreatedOrg[]; + orgs: NewOrExistingOrg[]; failed: CreateOrgData[]; fileName: string; totalOrgs: number; + existing: Partial[]; }> { + const { includeExistingOrgsInOutput, noDuplicateNames } = options; const content = await loadFile(filePath); const orgsData: CreateOrgData[] = []; const failedOrgs: CreateOrgData[] = []; @@ -48,9 +128,6 @@ export async function createOrgs( } catch (e) { throw new Error(`Failed to parse organizations from ${filePath}`); } - const requestManager = new requestsManager({ - userAgentPrefix: 'snyk-api-import', - }); debug(`Loaded ${orgsData.length} organizations to create ${Date.now()}`); const orgsPerGroup: { @@ -66,72 +143,57 @@ export async function createOrgs( } }); - const createdOrgs: CreatedOrg[] = []; + const createdOrgs: NewOrExistingOrg[] = []; + const existingOrgs: Org[] = []; + const requestManager = new requestsManager({ + userAgentPrefix: 'snyk-api-import', + }); for (const groupId in orgsPerGroup) { let orgsToCreate = orgsPerGroup[groupId]; - if (skipIfOrgNameExists) { - const { newOrgs, existingOrgs } = await filterOutExistingOrgs( - requestManager, - orgsData, - groupId, + const res = await filterOutExistingOrgs(requestManager, orgsData, groupId); + existingOrgs.push(...res.existingOrgs); + + if (noDuplicateNames) { + orgsToCreate = res.newOrgs; + failedOrgs.push( + ...res.existingOrgs.map((o) => ({ + groupId: o.group.id, + name: o.name, + })), ); - orgsToCreate = newOrgs; - failedOrgs.push(...existingOrgs); - if (existingOrgs.length > 0) { - console.log( - `Skipped creating ${ - existingOrgs.length - } organization(s) as the names were already used in the Group ${groupId}. Organizations skipped: ${existingOrgs - .map((o) => o.name) - .join(', ')}`, - ); - } } + debug(`Creating ${orgsToCreate.length} new organizations`); - for (const orgData of orgsToCreate) { - const { name, sourceOrgId } = orgData; - try { - const org = await createOrg(requestManager, groupId, name, sourceOrgId); - const integrations = - (await listIntegrations(requestManager, org.id)) || {}; - await setNotificationPreferences(requestManager, org.id, orgData); - createdOrgs.push({ - ...org, - orgId: org.id, - integrations, - groupId, - origName: name, - sourceOrgId, - }); - logCreatedOrg(groupId, name, org, integrations, loggingPath); - } catch (e) { - failedOrgs.push({ groupId, name, sourceOrgId }); - const errorMessage = e.data ? e.data.message : e.message; - logFailedOrg( - groupId, - name, - errorMessage || - 'Failed to create org, please try again in DEBUG mode.', - ); - debug( - `Failed to create organization with data: ${JSON.stringify( - orgsData, - )}`, - e, - ); - } - } + const { failed, created } = await createNewOrgs( + loggingPath, + requestManager, + groupId, + orgsToCreate, + ); + failedOrgs.push(...failed); + createdOrgs.push(...created); } - if (failedOrgs.length === orgsData.length) { + if (createdOrgs.length === 0) { throw new Error( - `All requested organizations failed to be created. Review the errors in ${path.resolve(__dirname, loggingPath)}/.${FAILED_ORG_LOG_NAME}`, + `All requested organizations failed to be created. Review the errors in ${path.resolve( + __dirname, + loggingPath, + )}/.${FAILED_ORG_LOG_NAME}`, ); } - const fileName = await saveCreatedOrgData(createdOrgs); + debug(`Getting existing ${existingOrgs.length} orgs data`); + const { existing } = await listExistingOrgsData(requestManager, existingOrgs); + debug('Saving results'); + const allOrgs: Partial[] = [...createdOrgs]; + if (includeExistingOrgsInOutput) { + allOrgs.push(...existing); + } + const fileName = await saveCreatedOrgData(allOrgs); return { orgs: createdOrgs, + existing: includeExistingOrgsInOutput ? existing : [], failed: failedOrgs, fileName, totalOrgs: orgsData.length, diff --git a/test/lib/org.test.ts b/test/lib/org.test.ts index 820ad808..bcc4aaf5 100644 --- a/test/lib/org.test.ts +++ b/test/lib/org.test.ts @@ -21,10 +21,7 @@ describe('Org notification settings', () => { const res = await setNotificationPreferences( requestManager, ORG_ID, - { - groupId: 'exampleGroupId', - name: 'exampleName', - }, + 'exampleName', { 'test-limit': { enabled: false, @@ -38,10 +35,11 @@ describe('Org notification settings', () => { }); }, 5000); it('Default disables all notifications', async () => { - const res = await setNotificationPreferences(requestManager, ORG_ID, { - groupId: 'exampleGroupId', - name: 'exampleName', - }); + const res = await setNotificationPreferences( + requestManager, + ORG_ID, + 'exampleName', + ); expect(res).toEqual({ 'new-issues-remediations': { enabled: false, diff --git a/test/lib/orgs.test.ts b/test/lib/orgs.test.ts index b1e9d2d8..56360a1c 100644 --- a/test/lib/orgs.test.ts +++ b/test/lib/orgs.test.ts @@ -57,12 +57,13 @@ describe('Orgs API', () => { orgs, GROUP_ID, ); - expect(existingOrgs).toEqual([ - { - groupId: GROUP_ID, - name: ORG_NAME, - }, - ]); + expect(existingOrgs.filter((o) => o.name === ORG_NAME)[0]).toMatchObject({ + name: ORG_NAME, + id: expect.any(String), + slug: expect.any(String), + url: expect.any(String), + group: expect.any(Object), + }); expect(newOrgs).toEqual([ { groupId: GROUP_ID, diff --git a/test/scripts/create-orgs.test.ts b/test/scripts/create-orgs.test.ts index 2171766f..c99265ca 100644 --- a/test/scripts/create-orgs.test.ts +++ b/test/scripts/create-orgs.test.ts @@ -1,10 +1,15 @@ import * as path from 'path'; +import * as fs from 'fs'; + import { requestsManager } from 'snyk-request-manager'; import { CREATED_ORG_LOG_NAME } from '../../src/common'; -import { createOrg, deleteOrg } from '../../src/lib'; +import { deleteOrg } from '../../src/lib'; import { createOrgs } from '../../src/scripts/create-orgs'; import { deleteFiles } from '../delete-files'; +const ORG_NAME = process.env.TEST_ORG_NAME as string; +const GROUP_ID = process.env.TEST_GROUP_ID as string; + jest.unmock('snyk-request-manager'); jest.requireActual('snyk-request-manager'); @@ -24,11 +29,15 @@ describe('createOrgs script', () => { process.env = { ...OLD_ENV }; await deleteFiles(filesToDelete); }); + afterAll(async () => { + process.env = { ...OLD_ENV }; for (const orgId of createdOrgs) { await deleteOrg(requestManager, orgId); } }); + + // too flaky to run every time it('create 1 org', async () => { const importFile = path.resolve( __dirname + '/fixtures/create-orgs/1-org/1-org.json', @@ -37,11 +46,15 @@ describe('createOrgs script', () => { process.env.SNYK_LOG_PATH = logPath; filesToDelete.push(path.resolve(logPath + `/abc.${CREATED_ORG_LOG_NAME}`)); - const { fileName, orgs } = await createOrgs(importFile); + const { fileName, orgs, existing } = await createOrgs(importFile); + createdOrgs.push(...orgs.map((o) => o.orgId)); + const log = path.resolve(logPath, fileName); + + filesToDelete.push(log); expect(orgs).not.toBeNull(); - expect(orgs[0]).toEqual({ + expect(orgs[0]).toMatchObject({ created: expect.any(String), - groupId: 'd64abc45-b39a-48a2-9636-a4f62adbf09a', + groupId: GROUP_ID, id: expect.any(String), integrations: expect.any(Object), name: 'snyk-api-import-hello', @@ -52,9 +65,59 @@ describe('createOrgs script', () => { url: expect.any(String), group: expect.any(Object), }); - createdOrgs.push(orgs[0].orgId); - filesToDelete.push(path.resolve(logPath, fileName)); - }, 20000); + expect(existing).not.toBeNull(); + expect(existing.length >= 1).toBeTruthy(); + expect(existing.filter((o) => o.name === ORG_NAME)[0]).toMatchObject({ + // created: expect.any(String), not always there? flaky + groupId: GROUP_ID, + id: expect.any(String), + integrations: expect.any(Object), + name: ORG_NAME, + orgId: expect.any(String), + origName: ORG_NAME, + slug: expect.any(String), + url: expect.any(String), + group: expect.any(Object), + }); + // give file a little time to be finished to be written + await new Promise((r) => setTimeout(r, 1000)); + const logFile = fs.readFileSync(log, 'utf8'); + expect(logFile).toMatch(ORG_NAME); + }, 50000); + it('create 1 org and do not list existing', async () => { + const importFile = path.resolve( + __dirname + '/fixtures/create-orgs/1-org/1-org.json', + ); + const logPath = path.resolve(__dirname + '/fixtures/create-orgs/1-org/'); + process.env.SNYK_LOG_PATH = logPath; + filesToDelete.push(path.resolve(logPath + `/abc.${CREATED_ORG_LOG_NAME}`)); + + const { fileName, orgs, existing } = await createOrgs(importFile, { + includeExistingOrgsInOutput: false, + }); + const log = path.resolve(logPath, fileName); + createdOrgs.push(...orgs.map((o) => o.orgId)); + filesToDelete.push(log); + expect(orgs).not.toBeNull(); + expect(orgs[0]).toMatchObject({ + created: expect.any(String), + groupId: GROUP_ID, + id: expect.any(String), + integrations: expect.any(Object), + name: 'snyk-api-import-hello', + orgId: expect.any(String), + origName: 'snyk-api-import-hello', + sourceOrgId: undefined, + slug: expect.any(String), + url: expect.any(String), + group: expect.any(Object), + }); + expect(existing).toEqual([]); + // give file a little time to be finished to be written + await new Promise((r) => setTimeout(r, 1000)); + const logFile = fs.readFileSync(log, 'utf8'); + expect(logFile).not.toMatch(ORG_NAME); + }, 50000); it('creating an org with the same name as an org in the Group fails', async () => { const importFile = path.resolve( @@ -64,22 +127,45 @@ describe('createOrgs script', () => { __dirname + '/fixtures/create-orgs/unique-org/', ); process.env.SNYK_LOG_PATH = logPath; - const skipIfOrgNameExists = true; + const noDuplicateNames = true; + const includeExistingOrgsInOutput = true; // first create the org - const { fileName, orgs, failed, totalOrgs } = await createOrgs(importFile); + const { fileName, orgs, failed, totalOrgs, existing } = await createOrgs( + importFile, + ); + // cleanup + createdOrgs.push(orgs[0].orgId); + filesToDelete.push(path.resolve(logPath, fileName)); + filesToDelete.push(path.resolve(logPath + `/abc.${CREATED_ORG_LOG_NAME}`)); + expect(failed).toHaveLength(0); expect(orgs).toHaveLength(1); expect(totalOrgs).toEqual(1); + + expect(existing).not.toBeNull(); + expect(existing.length >= 1).toBeTruthy(); + expect(existing.filter((o) => o.name === ORG_NAME)[0]).toMatchObject({ + // created: expect.any(String), not always there? flaky + groupId: GROUP_ID, + id: expect.any(String), + integrations: expect.any(Object), + name: ORG_NAME, + orgId: expect.any(String), + origName: ORG_NAME, + slug: expect.any(String), + url: expect.any(String), + group: expect.any(Object), + }); + // try again but in stricter name check mode and expect a it to fail - expect(createOrgs(importFile, skipIfOrgNameExists)).rejects.toThrow( + expect( + createOrgs(importFile, { noDuplicateNames, includeExistingOrgsInOutput }), + ).rejects.toThrow( 'All requested organizations failed to be created. Review the errors in', ); - // cleanup - createdOrgs.push(orgs[0].orgId); - filesToDelete.push(path.resolve(logPath, fileName)); - filesToDelete.push(path.resolve(logPath + `/abc.${CREATED_ORG_LOG_NAME}`)); - }, 40000); + }, 70000); + it.todo('creating multiple orgs'); it('creating an org fails', async () => { const importFile = path.resolve( @@ -94,5 +180,5 @@ describe('createOrgs script', () => { expect(createOrgs(importFile)).rejects.toThrow( 'fails-to-create/.failed-to-create-orgs.log', ); - }, 1000); + }, 70000); }); diff --git a/test/system/__snapshots__/help.test.ts.snap b/test/system/__snapshots__/help.test.ts.snap index 176e74e7..2e887072 100644 --- a/test/system/__snapshots__/help.test.ts.snap +++ b/test/system/__snapshots__/help.test.ts.snap @@ -15,8 +15,10 @@ Commands: organizations and their projects to generate this. The generated file can be used to skip previously imported targets when running the \`import\` command - index.js orgs:create Create the Orgs in Snyk based on data file generated - with \`orgs:data\` command + index.js orgs:create Create the organizations in Snyk based on data file + generated with \`orgs:data\` command. Output generates + key data for created and existing organizations for + use to generate project import data. index.js orgs:data Generate data required for Orgs to be created via API by mirroring a given source. diff --git a/test/system/__snapshots__/orgs:create.test.ts.snap b/test/system/__snapshots__/orgs:create.test.ts.snap index 4d0b6a4e..09d72371 100644 --- a/test/system/__snapshots__/orgs:create.test.ts.snap +++ b/test/system/__snapshots__/orgs:create.test.ts.snap @@ -3,13 +3,17 @@ exports[`\`snyk-api-import help <...>\` Shows help text as expected 1`] = ` "index.js orgs:create -Create the Orgs in Snyk based on data file generated with \`orgs:data\` command +Create the organizations in Snyk based on data file generated with \`orgs:data\` +command. Output generates key data for created and existing organizations for +use to generate project import data. Options: - --version Show version number [boolean] - --help Show help [boolean] - --file Path to data file generated with \`orgs:data\` command - [required] - --noDuplicateNames Skip creating an organization if the given name is already - taken within the Group." + --version Show version number [boolean] + --help Show help [boolean] + --file Path to data file generated with \`orgs:data\` + command [required] + --noDuplicateNames Skip creating an organization if the given name + is already taken within the Group. + --includeExistingOrgsInOutput Log existing organization information as well + as newly created [default: true]" `;