Merge pull request #378 from snyk-tech-services/feat/syncCommand

feat: create sync command

README.md

@@ -31,6 +31,8 @@ If you need to adjust concurrency you can stop the script, change the concurrenc
 
 - [Contributing](.github/CONTRIBUTING.md)
 - [Kicking off an import](docs/import.md)
+- [Sync: detecting changes in monitored repos and updating Snyk projects](docs/sync.md)
+
 - Example workflows
   - [AWS automation example](docs/example-workflows/aws-automation-example.md)
 

docs/sync.md

@@ -0,0 +1,65 @@
+# Sync
+## Table of Contents
+- [Sync](#sync)
+  - [Table of Contents](#table-of-contents)
+  - [Prerequisites](#prerequisites)
+- [What will change?](#what-will-change)
+  - [Branches](#branches)
+- [Kick off sync](#kick-off-sync)
+  - [1. Set the env vars](#1-set-the-env-vars)
+  - [2. Download & run](#2-download--run)
+  - [Examples](#examples)
+    - [Github.com](#githubcom)
+- [Known limitations](#known-limitations)
+
+## Prerequisites
+You will need to have setup in advance:
+- your [Snyk organizations](docs/orgs.md) should exist and have projects
+- your Snyk organizations configured with some connection to SCM (Github/Gitlab/Bitbucket etc) as you will need the provide which integration sync should use to update projects.
+- you will need your Snyk API token, with correct scope & [admin access for all Organizations](https://snyk.docs.apiary.io/#reference/import-projects/import/import-targets). This command will perform project changes on users behalf (import, update project branch, deactivate projects). **Github Integration Note**: As Github is both an auth & integration, how the integration is done has an effect on usage:
+  - For users importing via [Github Snyk integration](https://docs.snyk.io/integrations/git-repository-scm-integrations/github-integration#setting-up-a-github-integration) use your **personal Snyk API token** (Service Accounts are not supported for Github integration imports via API as this is a personal auth token only accessible to the user)
+  - For Github Enterprise Snyk integration with a url & token (for Github.com, Github Enterprise Cloud & Github Enterprise hosted) use a **Snyk API service account token**
+
+
+Any logs will be generated at `SNYK_LOG_PATH` directory.
+
+# What will change?
+
+## Branches
+Updating the project branch in Snyk to match the default branch of the repo in the SCM. The drift can happen for several reasons:
+- branch was renamed in Github/Gitlab etc on a repo from e.g. from  `master` > `main`
+- a new default branch was chosen from existing branches e.g. both `main` and `develop` exist as branches and default branch switched from `main` to `develop`
+
+
+# Kick off sync
+`sync` command will analyze existing projects & targets (repos) in Snyk organization and determine if any changes are needed.
+
+`--dryRun=true` - run the command first in dry-run mode to see what changes will be made in Snyk before running this again without if everything looks good. In this mode the last call to Snyk APIs to make the changes will be skipped but the logs will pretend as if it succeeded, the log entry will indicate this was generate in `dryRun` mode.
+
+The command will produce detailed logs for projects that were `updated` and those that needed an update but `failed`. If no changes are needed these will not be logged.
+
+
+## 1. Set the env vars
+- `SNYK_TOKEN` - your [Snyk api token](https://app.snyk.io/account)
+- `SNYK_LOG_PATH` - the path to folder where all logs should be saved,it is recommended creating a dedicated logs folder per import you have running. (Note: all logs will append)
+- `SNYK_API` (optional) defaults to `https://snyk.io/api/v1`
+- `GITHUB_TOKEN` - SCM token that has read level or similar permissions to see information about repos like default branch & can list files in a repo
+
+## 2. Download & run
+
+Grab a binary from the [releases page](https://github.com/snyk-tech-services/snyk-api-import/releases) and run with `DEBUG=snyk* snyk-api-import-macos import --file=path/to/imported-targets.json`
+
+
+## Examples
+
+### Github.com
+
+In dry-run mode:
+`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId=<snyk_org_public_id> --source=github --dryRun=true`
+
+Live mode:
+`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId=<snyk_org_public_id> --source=github`
+
+# Known limitations
+- Any organizations using a custom branch feature are currently not supported, `sync` will not continue.
+- ANy organizations that previously used the custom feature flag should ideally delete all existing projects & re-import to restore the project names to standard format (do not include a branch in the project name). `sync` will work regardless but may cause confusion as the project name will reference a branch that is not likely to be the actual branch being tested.

src/cmds/sync.ts

@@ -0,0 +1,99 @@
+import * as debugLib from 'debug';
+import * as yargs from 'yargs';
+const debug = debugLib('snyk:orgs-data-script');
+
+import { getLoggingPath } from '../lib/get-logging-path';
+import type { CommandResult } from '../lib/types';
+import { SupportedIntegrationTypesUpdateProject } from '../lib/types';
+
+import { updateOrgTargets } from '../scripts/sync/sync-org-projects';
+
+export const command = ['sync'];
+export const desc =
+  'Sync targets (e.g. repos) and their projects between Snyk and SCM for a given organization. Actions include:\n - updating monitored branch in Snyk to match the default branch from SCM';
+export const builder = {
+  orgPublicId: {
+    required: true,
+    default: undefined,
+    desc: 'Public id of the organization in Snyk that will be updated',
+  },
+  sourceUrl: {
+    required: false,
+    default: undefined,
+    desc: 'Custom base url for the source API that can list organizations (e.g. Github Enterprise url)',
+  },
+  // TODO: needs integration Type for GHE<> Github setup
+  source: {
+    required: true,
+    default: SupportedIntegrationTypesUpdateProject.GITHUB,
+    choices: [...Object.values(SupportedIntegrationTypesUpdateProject)],
+    desc: 'List of sources to be synced e.g. Github, Github Enterprise, Gitlab, Bitbucket Server, Bitbucket Cloud',
+  },
+  dryRun: {
+    required: false,
+    default: false,
+    desc: 'Dry run option. Will create a log file listing the potential updates',
+  },
+};
+
+export async function syncOrg(
+  source: SupportedIntegrationTypesUpdateProject[],
+  orgPublicId: string,
+  sourceUrl?: string,
+  dryRun?: boolean,
+): Promise<CommandResult> {
+  try {
+    getLoggingPath();
+
+    const res = await updateOrgTargets(orgPublicId, source, dryRun, sourceUrl);
+
+    const nothingToUpdate =
+      res.processedTargets == 0 &&
+      res.meta.projects.updated.length == 0 &&
+      res.meta.projects.failed.length == 0;
+    const orgMessage = nothingToUpdate
+      ? `Did not detect any changes to apply`
+      : `Processed ${res.processedTargets} targets\nUpdated ${res.meta.projects.updated.length} projects\n${res.meta.projects.failed.length} projects failed to update\nFind more information in ${res.fileName} and ${res.failedFileName}`;
+
+    return {
+      fileName: res.fileName,
+      exitCode: 0,
+      message:
+        `Finished syncing all ${source} targets for Snyk organization ${orgPublicId}\n` +
+        orgMessage,
+    };
+  } catch (e) {
+    const errorMessage = `ERROR! Failed to sync organization. Try running with \`DEBUG=snyk* <command> for more info\`.\nERROR: ${e.message}`;
+    return {
+      fileName: undefined,
+      exitCode: 1,
+      message: errorMessage,
+    };
+  }
+}
+
+export async function handler(argv: {
+  source: SupportedIntegrationTypesUpdateProject;
+  orgPublicId: string;
+  sourceUrl?: string;
+  dryRun?: boolean;
+}): Promise<void> {
+  const { source, orgPublicId, sourceUrl, dryRun } = argv;
+  debug('ℹ️  Options: ' + JSON.stringify(argv));
+
+  const sourceList: SupportedIntegrationTypesUpdateProject[] = [];
+  sourceList.push(source);
+
+  // when the input will be a file we will need to
+  // add a function to read and parse the file
+  const res = await syncOrg(sourceList, orgPublicId, sourceUrl, dryRun);
+
+  if (res.exitCode === 1) {
+    debug('Failed to sync organizations.\n' + res.message);
+
+    console.error(res.message);
+    setTimeout(() => yargs.exit(1, new Error(res.message)), 3000);
+  } else {
+    console.log(res.message);
+  }
+}

src/common.ts

@@ -2,13 +2,15 @@ export const IMPORT_LOG_NAME = 'imported-targets.log';
 export const FAILED_LOG_NAME = 'failed-imports.log';
 export const FAILED_PROJECTS_LOG_NAME = 'failed-projects.log';
 export const UPDATED_PROJECTS_LOG_NAME = 'updated-projects.log';
+export const FAILED_UPDATE_PROJECTS_LOG_NAME = 'failed-to-update-projects.log';
 export const FAILED_POLLS_LOG_NAME = 'failed-polls.log';
 export const IMPORT_JOBS_LOG_NAME = 'import-jobs.log';
 export const IMPORTED_PROJECTS_LOG_NAME = 'imported-projects.log';
 export const IMPORTED_BATCHES_LOG_NAME = 'imported-batches.log';
 export const IMPORT_JOB_RESULTS = 'import-job-results.log';
 export const CREATED_ORG_LOG_NAME = 'created-orgs.log'
 export const FAILED_ORG_LOG_NAME = 'failed-to-create-orgs.log'
+export const FAILED_SYNC_LOG_NAME = 'failed-to-sync-target.log'
 export const targetProps = [
   'name',
   'appId',

src/lib/api/project/index.ts

@@ -5,7 +5,7 @@ import { getApiToken } from '../../get-api-token';
 import { getSnykHost } from '../../get-snyk-host';
 import type { requestsManager } from 'snyk-request-manager';
 import type { SnykProject } from '../../types';
-const debug = debugLib('snyk:api-import');
+const debug = debugLib('snyk:api-project');
 
 export async function deleteProjects(
   orgId: string,
@@ -88,6 +88,7 @@ export async function updateProject(
 
   const statusCode = res.statusCode || res.status;
   if (!statusCode || statusCode !== 200) {
+    debug('Failed updating project: ' + projectId);
     throw new Error(
       'Expected a 200 response, instead received: ' +
         JSON.stringify({ data: res.data, status: statusCode }),

src/lib/project/update-branch.ts

@@ -31,7 +31,7 @@ export async function updateBranch(
     return { updated };
   } catch (e) {
     throw new Error(
-      `Failed to update project ${projectPublicId}. ERROR: ${e.message}`,
+      `Failed to update project ${projectPublicId} via Snyk API. ERROR: ${e.message}`,
     );
   }
 }

src/lib/source-handlers/github/get-default-branch.ts

@@ -14,7 +14,7 @@ export async function getGithubReposDefaultBranch(
   const baseUrl = getGithubBaseUrl(host);
   const octokit: Octokit = new Octokit({ baseUrl, auth: githubToken });
 
-  debug(`Fetch default branch for repo: ${target.owner}/${target.name}`);
+  debug(`Fetching default branch for repo: ${target.owner}/${target.name}`);
 
   const response = await octokit.repos.get({
     owner: target.owner!,

src/lib/source-handlers/github/index.ts

@@ -3,3 +3,4 @@ export * from './list-repos';
 export * from './organization-is-empty';
 export * from './get-default-branch';
 export * from './types';
+export * from './is-configured';

src/lib/source-handlers/github/is-configured.ts

@@ -0,0 +1,6 @@
+import { getGithubToken } from './get-github-token';
+
+export function isGithubConfigured(): boolean {
+  getGithubToken();
+  return true;
+}

src/loggers/log-failed-projects.ts

@@ -2,7 +2,7 @@ import * as bunyan from 'bunyan';
 import * as debugLib from 'debug';
 
 import { FAILED_PROJECTS_LOG_NAME } from './../common';
-import { Project } from './../lib/types';
+import type { Project } from './../lib/types';
 import { getLoggingPath } from './../lib';
 
 const debug = debugLib('snyk:import-projects-script');

src/loggers/log-failed-sync.ts

@@ -0,0 +1,32 @@
+import * as debugLib from 'debug';
+import * as bunyan from 'bunyan';
+
+import { getLoggingPath } from '../lib';
+import { FAILED_SYNC_LOG_NAME } from '../common';
+import type { SnykTarget } from '../lib/types';
+
+const debug = debugLib('snyk:failed-to-sync-orgs');
+
+export async function logFailedSync(
+  orgId: string,
+  target: SnykTarget,
+  errorMessage: string,
+  loggingPath = getLoggingPath(),
+): Promise<void> {
+  const log = bunyan.createLogger({
+    name: 'sync',
+    level: 'error',
+    streams: [
+      {
+        level: 'error',
+        path: `${loggingPath}/${orgId}.${FAILED_SYNC_LOG_NAME}`,
+      },
+    ],
+  });
+  try {
+    log.error({ orgId, target, errorMessage }, `Failed to sync target`);
+  } catch (e) {
+    debug('Failed to log failed sync', { orgId, target, errorMessage });
+    // do nothing
+  }
+}

src/loggers/log-failed-to-update-projects.ts

@@ -0,0 +1,46 @@
+import * as bunyan from 'bunyan';
+
+import { FAILED_UPDATE_PROJECTS_LOG_NAME } from './../common';
+import { getLoggingPath } from './../lib';
+import type { ProjectUpdateFailure } from '../scripts/sync/sync-projects-per-target';
+
+export async function logFailedToUpdateProjects(
+  orgId: string,
+  updated: ProjectUpdateFailure[],
+  loggingPath: string = getLoggingPath(),
+): Promise<void> {
+  try {
+    const log = bunyan.createLogger({
+      name: 'snyk:sync-org-projects',
+      level: 'info',
+      streams: [
+        {
+          level: 'info',
+          path: `${loggingPath}/${orgId}.${FAILED_UPDATE_PROJECTS_LOG_NAME}`,
+        },
+      ],
+    });
+    updated.forEach((update) => {
+      log.info(
+        {
+          orgId,
+          projectPublicId: update.projectPublicId,
+          from: update.from,
+          to: update.to,
+          update: update.type,
+          dryRun: String(update.dryRun),
+          target: {
+            id: update.target?.id,
+            origin: update.target?.attributes.origin,
+            displayName: update.target?.attributes.displayName,
+            remoteUrl: update.target?.attributes.remoteUrl ?? undefined,
+          },
+          error: update.errorMessage,
+        },
+        `Snyk project ${update.type} update failed`,
+      );
+    });
+  } catch (e) {
+    // do nothing
+  }
+}

src/loggers/log-updated-project.ts

@@ -1,19 +1,12 @@
 import * as bunyan from 'bunyan';
-import * as debugLib from 'debug';
 
 import { UPDATED_PROJECTS_LOG_NAME } from './../common';
-import { Project } from './../lib/types';
 import { getLoggingPath } from './../lib';
-
-const debug = debugLib('snyk:sync-org-projects');
-
-export interface FailedProject extends Project {
-  locationUrl: string;
-}
+import type { ProjectUpdate } from '../scripts/sync/sync-projects-per-target';
 
 export async function logUpdatedProjects(
   orgId: string,
-  branchesUpdated: string[],
+  updated: ProjectUpdate[],
   loggingPath: string = getLoggingPath(),
 ): Promise<void> {
   try {
@@ -27,14 +20,23 @@ export async function logUpdatedProjects(
         },
       ],
     });
-    branchesUpdated.forEach((projectPublicId) => {
-      debug(
-        { orgId, projectPublicId },
-        'Snyk project branch updated to point to default',
-      );
+    updated.forEach((update) => {
       log.info(
-        { orgId, projectPublicId },
-        'Snyk project branch updated to point to default',
+        {
+          orgId,
+          projectPublicId: update.projectPublicId,
+          from: update.from,
+          to: update.to,
+          update: update.type,
+          dryRun: String(update.dryRun),
+          target: {
+            id: update.target?.id,
+            origin: update.target?.attributes.origin,
+            displayName: update.target?.attributes.displayName,
+            remoteUrl: update.target?.attributes.remoteUrl ?? undefined,
+          },
+        },
+        `Snyk project ${update.type} updated`,
       );
     });
   } catch (e) {