feat: bitbucket-cloud command - wip

src/cmds/bitbucket-cloud.ts

@@ -0,0 +1,104 @@
+import * as debugLib from 'debug';
+import { BitbucketCloudTarget, ContributorMap } from '../lib/types';
+import { fetchBitbucketCloudContributors } from '../lib/bitbucket-cloud/bitbucket-cloud-contributors';
+import { SCMHandlerClass } from '../lib/common/SCMHandler';
+import { SourceType } from '../lib/snyk';
+
+const debug = debugLib('snyk:bitbucket-cloud-count');
+const bitbucketCLoudDefaultUrl = 'https://bitbucket.org/';
+export const command = ['bitbucket-cloud'];
+export const desc = 'Count contributors for bitbucket-cloud.\n';
+
+export const builder = {
+  user: {
+    required: true,
+    default: undefined,
+    desc: 'Bitbucket cloud username',
+  },
+  password: {
+    required: true,
+    default: undefined,
+    desc: 'Bitbucket cloud password',
+  },
+  workspaces: {
+    required: false,
+    default: undefined,
+    desc: 'Bitbucket cloud workspace name/uuid to count contributors for',
+  },
+  repo: {
+    required: false,
+    default: undefined,
+    desc: '[Optional] Specific repo to count only for',
+  },
+  exclusionFilePath: {
+    required: false,
+    default: undefined,
+    desc: '[Optional] Exclusion list filepath',
+  },
+  json: {
+    required: false,
+    desc: '[Optional] JSON output',
+  },
+  skipSnykMonitoredRepos: {
+    required: false,
+    desc: '[Optional] Skip Snyk monitored repos and count contributors for all repos',
+  },
+};
+
+class BitbucketCloud extends SCMHandlerClass {
+  bitbucketCloudConnInfo: BitbucketCloudTarget;
+  constructor(bitbucketCloudInfo: BitbucketCloudTarget) {
+    super();
+    this.bitbucketCloudConnInfo = bitbucketCloudInfo;
+  }
+
+  async fetchSCMContributors(
+    SnykMonitoredRepos: string[],
+  ): Promise<ContributorMap> {
+    let contributors: ContributorMap = new Map();
+    try {
+      debug('ℹ️  Options: ' + JSON.stringify(this.bitbucketCloudConnInfo));
+      contributors = await fetchBitbucketCloudContributors(
+        this.bitbucketCloudConnInfo,
+        SnykMonitoredRepos,
+      );
+      return contributors;
+    } catch (e) {
+      debug('Failed \n' + e);
+      console.error(`ERROR! ${e}`);
+    }
+    return contributors;
+  }
+}
+
+export async function handler(argv: {
+  user: string;
+  password: string;
+  workspaces?: string;
+  repo?: string;
+  exclusionFilePath: string;
+  json: boolean;
+  skipSnykMonitoredRepos: boolean;
+}): Promise<void> {
+  if (process.env.DEBUG) {
+    debug('DEBUG MODE ENABLED \n');
+    debug('ℹ️  Options: ' + JSON.stringify(argv));
+  }
+
+  const scmTarget: BitbucketCloudTarget = {
+    user: argv.user,
+    password: argv.password,
+    workspaces: argv.workspaces?.split(','),
+    repo: argv.repo,
+  };
+
+  const bitbucketCloudTask = new BitbucketCloud(scmTarget);
+
+  await bitbucketCloudTask.scmContributorCount(
+    bitbucketCLoudDefaultUrl,
+    SourceType['bitbucket-cloud'],
+    argv.skipSnykMonitoredRepos,
+    argv.exclusionFilePath,
+    argv.json,
+  );
+}

src/lib/bitbucket-cloud/bitbucket-cloud-contributors.ts

@@ -0,0 +1,183 @@
+import {
+  BitbucketCloudTarget,
+  Username,
+  Contributor,
+  ContributorMap,
+} from '../types';
+import { Commits, Repo } from './types';
+import { fetchAllPages, isAnyCommitMoreThan90Days } from './utils';
+
+import * as debugLib from 'debug';
+const bitbucketCloudDefaultUrl = 'https://bitbucket.org';
+const debug = debugLib('snyk:bitbucket-cloud-count');
+
+export const fetchBitbucketCloudContributors = async (
+  bitbucketCloudInfo: BitbucketCloudTarget,
+  SnykMonitoredRepos: string[],
+): Promise<ContributorMap> => {
+  const contributorsMap = new Map<Username, Contributor>();
+  try {
+    let repoList: Repo[] = [];
+
+    if (
+      bitbucketCloudInfo.repo &&
+      (!bitbucketCloudInfo.workspaces ||
+        bitbucketCloudInfo.workspaces.length > 1)
+    ) {
+      // If repo is specified, then a single project key is expected, bail otherwise
+      console.log('You must provide a single project for single repo counting');
+      process.exit(1);
+    } else if (bitbucketCloudInfo.repo) {
+      // If repo is specified, and we got a single project key
+      debug('Counting contributors for single repo');
+      if (bitbucketCloudInfo.workspaces) {
+        repoList.push({
+          slug: bitbucketCloudInfo.repo,
+          workspace: { uuid: bitbucketCloudInfo.workspaces[0] },
+        });
+      }
+    } else {
+      // Otherwise retrieve all repos (for given projects or all repos)
+      repoList = repoList.concat(
+        await fetchBitbucketCloudReposForWorkspaces(bitbucketCloudInfo),
+      );
+    }
+
+    if (SnykMonitoredRepos && SnykMonitoredRepos.length > 0) {
+      repoList = repoList.filter(
+        (repo) =>
+          SnykMonitoredRepos.includes(`${repo.workspace.uuid}/${repo.slug}`) ||
+          SnykMonitoredRepos.includes(`${repo.workspace.slug}/${repo.slug}`),
+      );
+    }
+
+    for (let i = 0; i < repoList.length; i++) {
+      await fetchBitbucketCloudContributorsForRepo(
+        bitbucketCloudInfo,
+        repoList[i],
+        contributorsMap,
+      );
+    }
+    debug(contributorsMap);
+    return contributorsMap;
+  } catch (err) {
+    debug('Failed to retrieve contributors from bitbucket-cloud.\n' + err);
+    console.log(
+      'Failed to retrieve contributors from bitbucket-cloud. Try running with `DEBUG=snyk* snyk-contributor`',
+    );
+  }
+  debug(contributorsMap);
+  return contributorsMap;
+};
+
+export const fetchBitbucketCloudContributorsForRepo = async (
+  bitbucketCloudInfo: BitbucketCloudTarget,
+  repo: Repo,
+  contributorsMap: ContributorMap,
+): Promise<void> => {
+  const fullUrl = `${bitbucketCloudDefaultUrl}/api/2.0/repositories/${repo.workspace.uuid}/${repo.slug}/commits`;
+  try {
+    debug(
+      `Fetching single repo contributor from bitbucket-cloud. Worspace ${repo.workspace.uuid} - Repo ${repo.slug}\n`,
+    );
+
+    // 7776000000 == 90 days in ms
+
+    const response = await fetchAllPages(
+      fullUrl,
+      bitbucketCloudInfo.user,
+      bitbucketCloudInfo.password,
+      isAnyCommitMoreThan90Days,
+    ) as Commits[];
+    // const result = await response.text();
+    // const parsedResponse = JSON.parse(result).values as Commits[];
+
+    const date: Date = new Date();
+    let today = date.getTime();
+    if (process.env.NODE_ENV == 'test') {
+      today = date.setFullYear(2020, 6, 15);
+    }
+    for (let i = 0; i < response.length; i++) {
+      const commit = response[i];
+      if(commit.author.user && commit.author.user.display_name){
+      commit.author.displayName = commit.author.user.display_name;
+      }
+      if(commit.author.raw){
+      commit.author.emailAddress = commit.author.raw.split('<')[1].split('>')[0];
+      }
+      if(commit.author.user && commit.author.user.nickname){
+      commit.author.name = commit.author.user.nickname;
+      }
+      // > is the right way, < is for testing really
+      // todayDate - 90 days should be smaller than commit timestamp, otherwise timestamp occured before 90days
+      const epochDateFromCommit = new Date(commit.date).getTime(); //((new Date(commit.date)).getTime())) 
+      if (today - 7776000000 > epochDateFromCommit) {
+        // Skipping if more than 90 days old
+        continue;
+      }
+      let contributionsCount = 1;
+      let reposContributedTo = [
+        `${repo.workspace.slug || repo.workspace.uuid}/${repo.slug}`,
+      ];
+
+      if (contributorsMap && contributorsMap.has(commit.author.name)) {
+        contributionsCount =
+          contributorsMap.get(commit.author.name)?.contributionsCount || 0;
+        contributionsCount++;
+
+        reposContributedTo =
+          contributorsMap.get(commit.author.name)?.reposContributedTo || [];
+        if (
+          !reposContributedTo.includes(
+            `${repo.workspace.slug || repo.workspace.uuid}/${repo.slug}`,
+          )
+        ) {
+          // Dedupping repo list here
+          reposContributedTo.push(`${repo.workspace.slug}/${repo.slug}`);
+        }
+      }
+      contributorsMap.set(commit.author.name, {
+        email: commit.author.emailAddress,
+        contributionsCount: contributionsCount,
+        reposContributedTo: reposContributedTo,
+      });
+    }
+  } catch (err) {
+    debug('Failed to retrieve commits from bitbucket-cloud.\n' + err);
+    console.log(
+      'Failed to retrieve commits from bitbucket-cloud. Try running with `DEBUG=snyk* snyk-contributor`',
+    );
+  }
+};
+
+export const fetchBitbucketCloudReposForWorkspaces = async (
+  bitbucketCloudInfo: BitbucketCloudTarget,
+): Promise<Repo[]> => {
+  let repoList: Repo[] = [];
+
+  const fullUrlSet: string[] = !bitbucketCloudInfo.workspaces
+    ? [`${bitbucketCloudDefaultUrl}/api/2.0/repositories?q=is_private=true&role=member`]
+    : bitbucketCloudInfo.workspaces.map(
+        (workspace) =>
+          `${bitbucketCloudDefaultUrl}/api/2.0/repositories/${workspace}`,
+      );
+      try {
+        for (let i = 0; i < fullUrlSet.length; i++) {
+          debug(`Fetching repos list ${fullUrlSet[i]}\n`);
+          repoList = repoList.concat(
+            (await fetchAllPages(
+              fullUrlSet[i],
+              bitbucketCloudInfo.user,
+              bitbucketCloudInfo.password,
+            )) as Repo[],
+          );
+        }
+        return repoList;
+      } catch (err) {
+        debug('Failed to retrieve repo list from bitbucket-cloud.\n' + err);
+        console.log(
+          'Failed to retrieve repo list from bitbucket-cloud. Try running with `DEBUG=snyk* snyk-contributor`',
+        );
+      }
+      return repoList;
+    };

src/lib/bitbucket-cloud/types.ts

@@ -0,0 +1,37 @@
+export interface CommitsApiResponse {
+  values: Commits[];
+}
+export interface Commits {
+  author: author;
+  date: number;
+}
+
+export interface author {
+  raw: string;
+  name: string;
+  emailAddress: string;
+  displayName: string;
+  user: user;
+}
+
+interface user {
+display_name: string;
+type: string;
+nickname: string;
+}
+
+export interface repoListApiResponse {
+  size: number;
+  isLastPage: boolean;
+  next?: string;
+  values: unknown[];
+}
+
+export interface Repo {
+  slug: string;
+  workspace: {
+    uuid: string;
+    slug?: string;
+  };
+  is_private?: boolean;
+}

src/lib/bitbucket-cloud/utils/index.ts

@@ -0,0 +1,69 @@
+import fetch  from 'node-fetch';
+import * as debugLib from 'debug';
+import { repoListApiResponse, Commits } from '../types';
+import Bottleneck from 'bottleneck';
+import base64 = require('base-64');
+
+const debug = debugLib('snyk:bitbucket-cloud-count');
+
+const limiter = new Bottleneck({
+  maxConcurrent: 1,
+  minTime: 333,
+});
+
+limiter.on('failed', async (error, jobInfo) => {
+  const id = jobInfo.options.id;
+  console.warn(`Job ${id} failed: ${error}`);
+  if (jobInfo.retryCount === 0) {
+    // Here we only retry once
+    console.log(`Retrying job ${id} in 25ms!`);
+    return 25;
+  }
+});
+
+export const isAnyCommitMoreThan90Days = (values: unknown[]): boolean => {
+  const date: Date = new Date();
+  if (process.env.NODE_ENV == 'test') {
+    date.setFullYear(2020, 6, 15);
+  }
+
+  const typedValues = values as Commits[];
+  // return true to break pagination if any commit if more than 90 days old
+  return typedValues.some(
+    (typedValue) => date.getTime() - 7776000000 > typedValue.date,
+  );
+};
+
+export const fetchAllPages = async (
+  url: string,
+  user: string,
+  password: string,
+  breakIfTrue?: (values: unknown[]) => boolean,
+): Promise<unknown[]> => {
+  let isLastPage = false;
+
+  let values: unknown[] = [];
+  let pageCount = 1;
+  while (!isLastPage) {
+    debug(`Fetching page ${pageCount}\n`);
+    const response = await limiter.schedule(() =>
+      fetch(`${url}`, {
+        method: 'GET',
+        headers: { Authorization: 'Basic ' + base64.encode(user + ":" + password) },
+      }),
+    );
+    const apiResponse = (await response.json()) as repoListApiResponse;
+    values = values.concat(apiResponse.values);
+    if (apiResponse.next){
+    url = apiResponse.next;
+    }
+    else{
+      isLastPage = true;
+    }
+    pageCount++;
+    if (breakIfTrue && breakIfTrue(values)) {
+      break;
+    }
+  }
+  return values;
+};