Added the ability to update labels based on configuration

src/commands/github/labels.ts

@@ -1,13 +1,28 @@
 import { flags } from '@oclif/command';
 import cli from 'cli-ux';
+import loadYamlFile from 'load-yaml-file';
+import * as path from 'path';
 
 import Command from '../../base';
 import fetchNodesByQuery from '../../utils/github/utils/fetchNodesByQuery';
 import ghClient from '../../utils/github/utils/ghClient';
 
+import checkConfig from '../../utils/github/labels/checkConfig';
+import { LabelsConfig } from '../../utils/github/labels/labelsConfig.type';
+import fetchAllLabels from '../../utils/github/labels/fetchAllLabels';
+
+import GQL_UPDATELABEL from '../../utils/github/labels/updateLabel.graphql';
+import GQL_RATELIMIT from '../../utils/import/getRateLimit.graphql';
+
 import esClient from '../../utils/es/esClient';
 import esGetActiveSources from '../../utils/es/esGetActiveSources';
 
+const sleep = (ms: number) => {
+  //https://github.com/Microsoft/tslint-microsoft-contrib/issues/355
+  // tslint:disable-next-line no-string-based-set-timeout
+  return new Promise(resolve => setTimeout(resolve, ms));
+};
+
 import {
   esMapping,
   esSettings,
@@ -25,6 +40,34 @@ import {
 
 import pushConfig from '../../utils/zencrepes/pushConfig';
 
+const findLabelConfig = (labelsConfig: LabelsConfig, label: any) => {
+  const configFound = labelsConfig.labels.find((l) => {
+    if (l.exactMatch === true && l.name === label.name) {
+      return true;
+    } else if (l.exactMatch === false && label.name.includes(l.name)) {
+      return true;
+    }
+    return false;
+  });
+  return configFound;
+};
+
+const checkRateLimit = async (rateLimit: any) => {
+  const resetAt = rateLimit.resetAt;
+  const remainingTokens = rateLimit.remaining;
+  if (remainingTokens <= 105 && resetAt !== null) {
+    console.log(
+      'Exhausted all available tokens, will resuming querying after ' +
+        new Date(resetAt * 1000),
+    );
+    const sleepDuration =
+      new Date(resetAt * 1000).getTime() - new Date().getTime();
+    console.log('Will resume querying in: ' + sleepDuration + 's');
+    await sleep(sleepDuration + 10000);
+    console.log('Ready to resume querying');
+  }
+};
+
 export default class Labels extends Command {
   static description = 'Github: Fetches labels attached to configured sources';
 
@@ -36,6 +79,11 @@ export default class Labels extends Command {
       description:
         'User Configuration passed as an environment variable, takes precedence over config file',
     }),
+    updateLabels: flags.boolean({
+      char: 'u',
+      default: false,
+      description: 'Updates labels based on the provided configuration',
+    }),    
     config: flags.boolean({
       char: 'c',
       default: false,
@@ -68,6 +116,117 @@ export default class Labels extends Command {
       return;
     }
 
+    if (flags.updateLabels === true) {
+      this.log('Update labels in bulk');
+      await checkConfig(this.config, this.log);
+      const labelsConfig: LabelsConfig = await loadYamlFile(
+        path.join(this.config.configDir, 'labels-config.yml'),
+      );
+
+      this.log('Grab existing labels from Elasticsearch');
+      const labelsIndex = userConfig.elasticsearch.dataIndices.githubLabels;
+      const labels: any[] = await fetchAllLabels(eClient, labelsIndex);
+
+      // Loop in existing labels to identify if they need to be updated
+      let data: any = {}; // eslint-disable-line
+      const labelsToUpdate = labels.reduce((acc, label) => {
+        const configFound = findLabelConfig(labelsConfig, label);
+        if (configFound !== undefined) {
+          const modifiedFields = []
+          if (label.description !== configFound.description) {
+            this.debug(`   Description: ${label.description} -> ${configFound.description}`);            
+            label = {
+              ...label,
+              description: configFound.description,
+            };
+            modifiedFields.push("description");
+          }
+          if (label.color !== configFound.color.toLocaleLowerCase()) {
+            this.debug(`   Color: ${label.color} -> ${configFound.color.toLocaleLowerCase()}`);            
+            label = {
+              ...label,
+              color: configFound.color.toLocaleLowerCase(),
+            };
+            modifiedFields.push("color");
+          }
+          if (modifiedFields.length > 0) {
+            this.log(`Repository: ${label.repository.url} - Label ${label.name} will be updated. Fields: ${modifiedFields.toString()}`);
+            acc.push(label);
+          }
+        }
+        return acc;
+      }, []);
+
+      // Loop in existing labels to identify if they need to be updated
+      this.log(`Found ${labelsToUpdate.length} labels to update`);
+
+      let cpt = 0;
+      for (const label of labelsToUpdate) {
+        cpt++;
+        // Checking rate limit every 100 requests
+        if (cpt === 100) {
+          try {
+            data = await gClient.query({
+              query: GQL_RATELIMIT,
+              fetchPolicy: 'no-cache',
+              errorPolicy: 'ignore',
+            });
+          } catch (error) {
+            console.log(JSON.stringify(GQL_RATELIMIT));
+            console.log('THIS IS AN ERROR');
+            this.log(error);
+          }
+          if (data.data.rateLimit !== undefined) {
+            this.log(
+              'GitHub Tokens - remaining: ' +
+                data.data.rateLimit.remaining +
+                ' query cost: ' +
+                data.data.rateLimit.cost +
+                ' (token will reset at: ' +
+                data.data.rateLimit.resetAt +
+                ')',
+            );
+            await checkRateLimit(data.data.rateLimit);
+          } else {
+            this.exit();
+          }
+          cpt = 0;
+        }
+
+        cli.action.start(`${cpt}/${labelsToUpdate.length} - Repository: ${label.repository.url} - Label: ${label.name}`);
+        try {
+          data = await gClient.query({
+            query: GQL_UPDATELABEL,
+            variables: { labelId: label.id, color: label.color, description: label.description },
+            fetchPolicy: 'no-cache',
+            errorPolicy: 'ignore',
+          });
+        } catch (error) {
+          console.log(JSON.stringify(GQL_UPDATELABEL));
+          console.log('THIS IS AN ERROR');
+          this.log(error);
+        }
+        await sleep(250);
+
+        if (
+          data.data !== undefined &&
+          data.data.errors !== undefined &&
+          data.data.errors.length > 0
+        ) {
+          data.data.errors.forEach((error: { message: string }) => {
+            this.log(error.message);
+          });
+        } else {
+          // If there was no errors, the label is pushed back to Elasticsearch
+          // This makes it possible to keep track of the labels that have been updated
+          // And allow for resuming where the process was interrupted (if interrupted)
+          await pushEsNodes(eClient, labelsIndex, [label], this.log, true);
+        }
+        cli.action.stop('done');
+      }
+      return;
+    }
+
     const sources = await esGetActiveSources(eClient, userConfig, 'GITHUB');
 
     const fetchData = new fetchNodesByQuery(

src/commands/utils/updateLinks.ts

@@ -73,10 +73,14 @@ export default class UpdateLinks extends Command {
         src: string;
         srcName: string;
         dst: string;
-      }[] = [];
+      }[] = [];      
       const findLinks = /\[(.+)\]\((https?:\/\/[^\s]+)(?: "(.+)")?\)|(https?:\/\/[^\s]+)/gi;
 
       const found = i.body.match(findLinks);
+      if (i.title.includes('SUP-417')) {
+        console.log(i.title);
+        console.log(found);
+      }
       if (found !== null) {
         for (const mkLink of found) {
           const urlEx = /\(([^)]+)\)/g;

src/components/esUtils/pushEsNodes.ts

@@ -18,17 +18,20 @@ const pushEsNodes = async (
   nodesIndex: string,
   fetchedNodes: Array<object>,
   logger: Function,
+  silent: boolean = false,
 ) => {
   const esPayloadChunked = await chunkArray(fetchedNodes, 100);
   // Push the data back to elasticsearch
   for (const [idx, esPayloadChunk] of esPayloadChunked.entries()) {
-    logger(
-      'Submitting data to ElasticSearch (' +
-        (idx + 1) +
-        ' / ' +
-        esPayloadChunked.length +
-        ')',
-    );
+    if (!silent) {
+      logger(
+        'Submitting data to ElasticSearch (' +
+          (idx + 1) +
+          ' / ' +
+          esPayloadChunked.length +
+          ')',
+      );
+    }
     let formattedData = '';
     for (const rec of esPayloadChunk) {
       formattedData =

src/utils/github/labels/checkConfig.ts

@@ -0,0 +1,34 @@
+import * as fs from 'fs';
+import * as fse from 'fs-extra';
+import * as jsYaml from 'js-yaml';
+import * as path from 'path';
+
+import defaultLabelsConfig from './defaultLabelsConfig';
+import LabelsConfig from './labelsConfig.type';
+
+const checkConfig = async (config: any, log: any) => {
+  const labelsConfig: LabelsConfig = defaultLabelsConfig;
+
+  // Ensure index exists in Elasticsearch
+  // If config file does not exists, initialize it:
+  fse.ensureDirSync(config.configDir);
+
+  if (!fs.existsSync(path.join(config.configDir, 'labels-config.yml'))) {
+    fs.writeFileSync(
+      path.join(config.configDir, 'labels-config.yml'),
+      jsYaml.safeDump(labelsConfig),
+    );
+    log(
+      'Initialized configuration file with defaults in: ' +
+        path.join(config.configDir, 'labels-config.yml'),
+    );
+    log('Please EDIT the configuration file first');
+  } else {
+    log(
+      'Configuration file exists: ' +
+        path.join(config.configDir, 'labels-config.yml'),
+    );
+  }
+};
+
+export default checkConfig;

src/utils/github/labels/defaultLabelsConfig.ts

@@ -0,0 +1,15 @@
+const defaultImportConfig = {
+  labels: [{
+    name: 'A Label',
+    description: 'Description for my label',
+    color: '#0e8a16',
+    exactMatch: true,
+  }, {
+    name: 'Another Label',
+    description: 'Description for my other label',
+    color: '#A0315F',
+    exactMatch: false,
+  }],
+};
+
+export default defaultImportConfig;

src/utils/github/labels/fetchAllLabels.ts

@@ -0,0 +1,25 @@
+import cli from 'cli-ux';
+import { Client } from '@elastic/elasticsearch';
+
+const fetchAllLabels = async (esClient: Client, esIndex: string) => {
+  let labels: any[] = [];
+
+  const scrollSearch = esClient.helpers.scrollSearch({
+    index: esIndex,
+    body: {
+      query: {
+        // eslint-disable-next-line @typescript-eslint/camelcase
+        match_all: {},
+      },
+    },
+  });
+  cli.action.start('Fetching all labels from: ' + esIndex);
+
+  for await (const result of scrollSearch) {
+    labels = [...labels, ...result.documents];
+  }
+  cli.action.stop('done (' + labels.length + ' labels fetched)');
+
+  return labels;
+};
+export default fetchAllLabels;

src/utils/github/labels/labelsConfig.type.ts

@@ -0,0 +1,12 @@
+export interface Label {
+  name: string;
+  description: string;
+  color: string;
+  exactMatch: boolean;
+}
+
+export interface LabelsConfig {
+  labels: Label[];
+}
+
+export default LabelsConfig;

src/utils/github/labels/updateLabel.graphql.ts

@@ -0,0 +1,13 @@
+import gql from 'graphql-tag';
+
+const GQL_QUERY = gql`
+  mutation($labelId: ID!, $color: String!, $description: String) {
+    updateLabel(input: { id: $labelId, color: $color, description: $description }) {
+      label {
+        id
+      }
+    }
+  }
+`;
+
+export default GQL_QUERY;