Merge pull request #188 from snyk-tech-services/feat/log-existing-orgs

feat: log existing orgs data
snyk · Apr 9, 2021 · 41f111b · 41f111b
2 parents 76e3bed + 9d21f87
commit 41f111b
Show file tree

Hide file tree

Showing 9 changed files with 285 additions and 113 deletions.
diff --git a/src/cmds/orgs:create.ts b/src/cmds/orgs:create.ts
@@ -6,7 +6,7 @@ import { createOrgs } from '../scripts/create-orgs';
 
 export const command = ['orgs:create'];
 export const desc =
-  'Create the Orgs in Snyk based on data file generated with `orgs:data` command';
+  'Create the organizations in Snyk based on data file generated with `orgs:data` command. Output generates key data for created and existing organizations for use to generate project import data.';
 export const builder = {
   file: {
     required: true,
@@ -18,17 +18,26 @@ export const builder = {
     desc:
       'Skip creating an organization if the given name is already taken within the Group.',
   },
+  includeExistingOrgsInOutput: {
+    required: false,
+    default: true,
+    desc: 'Log existing organization information as well as newly created',
+  },
 };
 
 export async function handler(argv: {
   file: string;
+  includeExistingOrgsInOutput: boolean;
   noDuplicateNames?: boolean;
 }): Promise<void> {
   try {
     getLoggingPath();
-    const { file, noDuplicateNames } = argv;
+    const { file, noDuplicateNames, includeExistingOrgsInOutput } = argv;
     debug('ℹ️  Options: ' + JSON.stringify(argv));
-    const res = await createOrgs(file, noDuplicateNames);
+    const res = await createOrgs(file, {
+      noDuplicateNames,
+      includeExistingOrgsInOutput,
+    });
 
     const orgsMessage =
       res.orgs.length > 0

diff --git a/src/lib/api/org/index.ts b/src/lib/api/org/index.ts
@@ -69,12 +69,12 @@ interface NotificationSettings {
 export async function setNotificationPreferences(
   requestManager: requestsManager,
   orgId: string,
-  orgData: CreateOrgData,
+  orgName: string,
   settings: NotificationSettings = defaultDisabledSettings,
 ): Promise<IntegrationsListResponse> {
   getApiToken();
   getSnykHost();
-  debug(`Disabling notifications for org: ${orgData.name} (${orgId})`);
+  debug(`Disabling notifications for org: ${orgName} (${orgId})`);
 
   if (!orgId) {
     throw new Error(

diff --git a/src/lib/filter-out-existing-orgs.ts b/src/lib/filter-out-existing-orgs.ts
@@ -1,13 +1,15 @@
+import * as _ from 'lodash';
+
 import { requestsManager } from 'snyk-request-manager';
 import { getAllOrgs } from './get-all-orgs-for-group';
-import { CreateOrgData } from './types';
+import { CreateOrgData, Org } from './types';
 
 export async function filterOutExistingOrgs(
   requestManager: requestsManager,
   orgs: CreateOrgData[] = [],
   groupId: string,
 ): Promise<{
-  existingOrgs: CreateOrgData[];
+  existingOrgs: Org[];
   newOrgs: CreateOrgData[];
 }> {
   if (!groupId) {
@@ -17,17 +19,25 @@ export async function filterOutExistingOrgs(
     return { existingOrgs: [], newOrgs: [] };
   }
 
-  const existingOrgs: CreateOrgData[] = [];
+  const existingOrgs: Org[] = [];
   const newOrgs: CreateOrgData[] = [];
   const groupOrgs = await getAllOrgs(requestManager, groupId);
   const uniqueOrgNames: Set<string> = new Set(groupOrgs.map((org) => org.name));
   for (const org of orgs) {
-    if (uniqueOrgNames.has(org.name)) {
-      existingOrgs.push(org);
-      continue;
+    if (!uniqueOrgNames.has(org.name)) {
+      newOrgs.push(org);
     }
-    newOrgs.push(org);
+  }
+  if (existingOrgs.length > 0) {
+    console.log(
+      `Skipped creating ${
+        existingOrgs.length
+      } organization(s) as the names were already used in the Group ${groupId}. Organizations skipped: ${existingOrgs
+        .map((o) => o.name)
+        .join(', ')}`,
+    );
   }
 
-  return { existingOrgs, newOrgs };
+  return { existingOrgs: groupOrgs, newOrgs };
 }
+
diff --git a/src/scripts/create-orgs.ts b/src/scripts/create-orgs.ts
@@ -6,14 +6,14 @@ import { CreatedOrgResponse, createOrg, filterOutExistingOrgs } from '../lib';
 import { getLoggingPath } from './../lib';
 import { listIntegrations, setNotificationPreferences } from '../lib/api/org';
 import { requestsManager } from 'snyk-request-manager';
-import { CreateOrgData } from '../lib/types';
+import { CreateOrgData, Org } from '../lib/types';
 import { logCreatedOrg } from '../loggers/log-created-org';
 import { writeFile } from '../write-file';
 import { FAILED_ORG_LOG_NAME } from '../common';
 import { logFailedOrg } from '../loggers/log-failed-org';
 
 const debug = debugLib('snyk:create-orgs-script');
-interface CreatedOrg extends CreatedOrgResponse {
+interface NewOrExistingOrg extends CreatedOrgResponse {
   integrations: {
     [name: string]: string;
   };
@@ -23,22 +23,102 @@ interface CreatedOrg extends CreatedOrgResponse {
   sourceOrgId?: string;
 }
 
-async function saveCreatedOrgData(orgData: CreatedOrg[]): Promise<string> {
+async function saveCreatedOrgData(
+  orgData: Partial<NewOrExistingOrg>[],
+): Promise<string> {
   const fileName = 'snyk-created-orgs.json';
   await writeFile(fileName, ({ orgData } as unknown) as JSON);
   return fileName;
 }
+async function createNewOrgs(
+  loggingPath: string,
+  requestManager: requestsManager,
+  groupId: string,
+  orgsToCreate: CreateOrgData[],
+): Promise<{ failed: CreateOrgData[]; created: NewOrExistingOrg[] }> {
+  const failed: CreateOrgData[] = [];
+  const created: NewOrExistingOrg[] = [];
+
+  for (const orgData of orgsToCreate) {
+    const { name, sourceOrgId } = orgData;
+    try {
+      const org = await createOrg(requestManager, groupId, name, sourceOrgId);
+      const integrations =
+        (await listIntegrations(requestManager, org.id)) || {};
+      await setNotificationPreferences(requestManager, org.id, org.name);
+      created.push({
+        ...org,
+        orgId: org.id,
+        integrations,
+        groupId,
+        origName: name,
+        sourceOrgId,
+      });
+      logCreatedOrg(groupId, name, org, integrations, loggingPath);
+    } catch (e) {
+      failed.push({ groupId, name, sourceOrgId });
+      const errorMessage = e.data ? e.data.message : e.message;
+      logFailedOrg(
+        groupId,
+        name,
+        errorMessage || 'Failed to create org, please try again in DEBUG mode.',
+      );
+      debug(
+        `Failed to create organization with data: ${JSON.stringify(orgData)}`,
+        e,
+      );
+    }
+  }
+
+  return { failed, created };
+}
+
+async function listExistingOrgsData(
+  requestManager: requestsManager,
+  existingOrgs: Org[],
+): Promise<{ existing: Partial<NewOrExistingOrg>[] }> {
+  const previouslyCreated: Partial<NewOrExistingOrg>[] = [];
+
+  for (const orgData of existingOrgs) {
+    const { name, id, group } = orgData;
+    try {
+      const integrations = (await listIntegrations(requestManager, id)) || {};
+      previouslyCreated.push({
+        ...orgData,
+        name,
+        orgId: id,
+        integrations,
+        groupId: group.id,
+        origName: name,
+      });
+    } catch (e) {
+      debug(
+        `Failed to list integrations for Org: ${orgData.name} (${orgData.id})`,
+        e,
+      );
+    }
+  }
+  return { existing: previouslyCreated };
+}
 
 export async function createOrgs(
   filePath: string,
-  skipIfOrgNameExists = false,
+  options: {
+    noDuplicateNames?: boolean;
+    includeExistingOrgsInOutput: boolean;
+  } = {
+    noDuplicateNames: false,
+    includeExistingOrgsInOutput: true,
+  },
   loggingPath = getLoggingPath(),
 ): Promise<{
-  orgs: CreatedOrg[];
+  orgs: NewOrExistingOrg[];
   failed: CreateOrgData[];
   fileName: string;
   totalOrgs: number;
+  existing: Partial<NewOrExistingOrg>[];
 }> {
+  const { includeExistingOrgsInOutput, noDuplicateNames } = options;
   const content = await loadFile(filePath);
   const orgsData: CreateOrgData[] = [];
   const failedOrgs: CreateOrgData[] = [];
@@ -48,9 +128,6 @@ export async function createOrgs(
   } catch (e) {
     throw new Error(`Failed to parse organizations from ${filePath}`);
   }
-  const requestManager = new requestsManager({
-    userAgentPrefix: 'snyk-api-import',
-  });
   debug(`Loaded ${orgsData.length} organizations to create ${Date.now()}`);
 
   const orgsPerGroup: {
@@ -66,72 +143,57 @@ export async function createOrgs(
     }
   });
 
-  const createdOrgs: CreatedOrg[] = [];
+  const createdOrgs: NewOrExistingOrg[] = [];
+  const existingOrgs: Org[] = [];
+  const requestManager = new requestsManager({
+    userAgentPrefix: 'snyk-api-import',
+  });
 
   for (const groupId in orgsPerGroup) {
     let orgsToCreate = orgsPerGroup[groupId];
-    if (skipIfOrgNameExists) {
-      const { newOrgs, existingOrgs } = await filterOutExistingOrgs(
-        requestManager,
-        orgsData,
-        groupId,
+    const res = await filterOutExistingOrgs(requestManager, orgsData, groupId);
+    existingOrgs.push(...res.existingOrgs);
+
+    if (noDuplicateNames) {
+      orgsToCreate = res.newOrgs;
+      failedOrgs.push(
+        ...res.existingOrgs.map((o) => ({
+          groupId: o.group.id,
+          name: o.name,
+        })),
       );
-      orgsToCreate = newOrgs;
-      failedOrgs.push(...existingOrgs);
-      if (existingOrgs.length > 0) {
-        console.log(
-          `Skipped creating ${
-            existingOrgs.length
-          } organization(s) as the names were already used in the Group ${groupId}. Organizations skipped: ${existingOrgs
-            .map((o) => o.name)
-            .join(', ')}`,
-        );
-      }
     }
+    debug(`Creating ${orgsToCreate.length} new organizations`);
 
-    for (const orgData of orgsToCreate) {
-      const { name, sourceOrgId } = orgData;
-      try {
-        const org = await createOrg(requestManager, groupId, name, sourceOrgId);
-        const integrations =
-          (await listIntegrations(requestManager, org.id)) || {};
-        await setNotificationPreferences(requestManager, org.id, orgData);
-        createdOrgs.push({
-          ...org,
-          orgId: org.id,
-          integrations,
-          groupId,
-          origName: name,
-          sourceOrgId,
-        });
-        logCreatedOrg(groupId, name, org, integrations, loggingPath);
-      } catch (e) {
-        failedOrgs.push({ groupId, name, sourceOrgId });
-        const errorMessage = e.data ? e.data.message : e.message;
-        logFailedOrg(
-          groupId,
-          name,
-          errorMessage ||
-            'Failed to create org, please try again in DEBUG mode.',
-        );
-        debug(
-          `Failed to create organization with data: ${JSON.stringify(
-            orgsData,
-          )}`,
-          e,
-        );
-      }
-    }
+    const { failed, created } = await createNewOrgs(
+      loggingPath,
+      requestManager,
+      groupId,
+      orgsToCreate,
+    );
+    failedOrgs.push(...failed);
+    createdOrgs.push(...created);
   }
 
-  if (failedOrgs.length === orgsData.length) {
+  if (createdOrgs.length === 0) {
     throw new Error(
-      `All requested organizations failed to be created. Review the errors in ${path.resolve(__dirname, loggingPath)}/<groupId>.${FAILED_ORG_LOG_NAME}`,
+      `All requested organizations failed to be created. Review the errors in ${path.resolve(
+        __dirname,
+        loggingPath,
+      )}/<groupId>.${FAILED_ORG_LOG_NAME}`,
     );
   }
-  const fileName = await saveCreatedOrgData(createdOrgs);
+  debug(`Getting existing ${existingOrgs.length} orgs data`);
+  const { existing } = await listExistingOrgsData(requestManager, existingOrgs);
+  debug('Saving results');
+  const allOrgs: Partial<NewOrExistingOrg>[] = [...createdOrgs];
+  if (includeExistingOrgsInOutput) {
+    allOrgs.push(...existing);
+  }
+  const fileName = await saveCreatedOrgData(allOrgs);
   return {
     orgs: createdOrgs,
+    existing: includeExistingOrgsInOutput ? existing : [],
     failed: failedOrgs,
     fileName,
     totalOrgs: orgsData.length,

diff --git a/test/lib/org.test.ts b/test/lib/org.test.ts
@@ -21,10 +21,7 @@ describe('Org notification settings', () => {
     const res = await setNotificationPreferences(
       requestManager,
       ORG_ID,
-      {
-        groupId: 'exampleGroupId',
-        name: 'exampleName',
-      },
+      'exampleName',
       {
         'test-limit': {
           enabled: false,
@@ -38,10 +35,11 @@ describe('Org notification settings', () => {
     });
   }, 5000);
   it('Default disables all notifications', async () => {
-    const res = await setNotificationPreferences(requestManager, ORG_ID, {
-      groupId: 'exampleGroupId',
-      name: 'exampleName',
-    });
+    const res = await setNotificationPreferences(
+      requestManager,
+      ORG_ID,
+      'exampleName',
+    );
     expect(res).toEqual({
       'new-issues-remediations': {
         enabled: false,

diff --git a/test/lib/orgs.test.ts b/test/lib/orgs.test.ts
@@ -57,12 +57,13 @@ describe('Orgs API', () => {
       orgs,
       GROUP_ID,
     );
-    expect(existingOrgs).toEqual([
-      {
-        groupId: GROUP_ID,
-        name: ORG_NAME,
-      },
-    ]);
+    expect(existingOrgs.filter((o) => o.name === ORG_NAME)[0]).toMatchObject({
+      name: ORG_NAME,
+      id: expect.any(String),
+      slug: expect.any(String),
+      url: expect.any(String),
+      group: expect.any(Object),
+    });
     expect(newOrgs).toEqual([
       {
         groupId: GROUP_ID,