Merge pull request #873 from snyk/feat/remote-rulesets-autosync

Feat/remote rulesets autosync [HYB-714]

lib/client/config/filters.ts

@@ -0,0 +1,11 @@
+import { LoadedFiltersSet } from '../../common/types/options';
+
+let globalFilterConfig: LoadedFiltersSet = {};
+
+export const getFilterConfig = () => {
+  return globalFilterConfig;
+};
+
+export const setFilterConfig = (filterConfig: LoadedFiltersSet) => {
+  globalFilterConfig = filterConfig;
+};

lib/client/index.ts

@@ -22,13 +22,15 @@ import {
 } from './hooks/startup/processHooks';
 import { forwardHttpRequestOverHttp } from '../common/relay/forwardHttpRequestOverHttp';
 import { isWebsocketConnOpen } from './utils/socketHelpers';
-import { loadAllFilters } from '../common/filter/filtersAsync';
-import { ClientOpts, LoadedClientOpts } from '../common/types/options';
+import { ClientOpts } from '../common/types/options';
 import { websocketConnectionSelectorMiddleware } from './routesHandler/websocketConnectionMiddlewares';
 import { getClientConfigMetadata } from './config/configHelpers';
 import { loadPlugins } from './brokerClientPlugins/pluginManager';
 import { manageWebsocketConnections } from './connectionsManager/manager';
 import { findPluginFolder } from '../common/config/config';
+import { retrieveAndLoadFilters } from './utils/filterLoading';
+
+const ONEDAY = 24 * 3600 * 1000; // 24h in ms
 
 process.on('uncaughtException', (error) => {
   if (error.message == 'read ECONNRESET') {
@@ -87,21 +89,17 @@ export const main = async (clientOpts: ClientOpts) => {
         clientOpts.config.brokerClientId,
       );
     }
-
-    const loadedClientOpts: LoadedClientOpts = {
-      loadedFilters: loadAllFilters(clientOpts.filters, clientOpts.config),
-      ...clientOpts,
-    };
-
-    if (!loadedClientOpts.loadedFilters) {
-      logger.error({ clientOpts }, 'Unable to load filters');
-      throw new Error('Unable to load filters');
+    await retrieveAndLoadFilters(clientOpts);
+    if (process.env.NODE_ENV != 'test') {
+      setInterval(async () => {
+        await retrieveAndLoadFilters(clientOpts);
+      }, ONEDAY);
     }
 
     const globalIdentifyingMetadata: IdentifyingMetadata = {
       capabilities: ['post-streams'],
       clientId: clientOpts.config.brokerClientId,
-      filters: loadedClientOpts.filters,
+      filters: clientOpts.filters,
       preflightChecks: hookResults.preflightCheckResults,
       version,
       clientConfig: getClientConfigMetadata(clientOpts.config),
@@ -111,48 +109,33 @@ export const main = async (clientOpts: ClientOpts) => {
     };
 
     let websocketConnections: WebSocketConnection[] = [];
-    if (loadedClientOpts.config.universalBrokerEnabled) {
+    if (clientOpts.config.universalBrokerEnabled) {
       websocketConnections = await manageWebsocketConnections(
-        loadedClientOpts,
+        clientOpts,
         globalIdentifyingMetadata,
       );
     } else {
       websocketConnections.push(
-        createWebSocket(
-          loadedClientOpts,
-          globalIdentifyingMetadata,
-          Role.primary,
-        ),
+        createWebSocket(clientOpts, globalIdentifyingMetadata, Role.primary),
       );
       websocketConnections.push(
-        createWebSocket(
-          loadedClientOpts,
-          globalIdentifyingMetadata,
-          Role.secondary,
-        ),
+        createWebSocket(clientOpts, globalIdentifyingMetadata, Role.secondary),
       );
     }
 
     // start the local webserver to listen for relay requests
-    const { app, server } = webserver(
-      loadedClientOpts.config,
-      loadedClientOpts.port,
-    );
-
-    const httpToWsForwarder = forwardHttpRequest(loadedClientOpts);
+    const { app, server } = webserver(clientOpts.config, clientOpts.port);
+    const httpToWsForwarder = forwardHttpRequest(clientOpts);
     const httpToAPIForwarder = forwardHttpRequestOverHttp(
-      loadedClientOpts,
-      loadedClientOpts.config,
+      clientOpts,
+      clientOpts.config,
     );
     // IMPORTANT: defined before relay (`app.all('/*', ...`)
-    app.get('/health/checks', handleChecksRoute(loadedClientOpts.config));
-    app.get(
-      '/health/checks/:checkId',
-      handleCheckIdsRoutes(loadedClientOpts.config),
-    );
+    app.get('/health/checks', handleChecksRoute(clientOpts.config));
+    app.get('/health/checks/:checkId', handleCheckIdsRoutes(clientOpts.config));
 
     app.get(
-      loadedClientOpts.config.brokerHealthcheckPath || '/healthcheck',
+      clientOpts.config.brokerHealthcheckPath || '/healthcheck',
       (req, res, next) => {
         res.locals.websocketConnections = websocketConnections;
         next();
@@ -161,13 +144,17 @@ export const main = async (clientOpts: ClientOpts) => {
     );
 
     app.get('/filters', (req, res) => {
-      res.send(loadedClientOpts.filters);
+      res.send(clientOpts.filters);
+    });
+    app.post('/filters', async (req, res) => {
+      await retrieveAndLoadFilters(clientOpts);
+      res.send(clientOpts.filters);
     });
 
     app.get(
-      loadedClientOpts.config.brokerSystemcheckPath || '/systemcheck',
+      clientOpts.config.brokerSystemcheckPath || '/systemcheck',
       (req, res, next) => {
-        res.locals.clientOpts = loadedClientOpts;
+        res.locals.clientOpts = clientOpts;
         next();
       },
       systemCheckHandler,

lib/client/utils/filterLoading.ts

@@ -0,0 +1,15 @@
+import filterRulesLoader from '../../common/filter/filter-rules-loading';
+import { CONFIGURATION } from '../../common/config/config';
+import { loadAllFilters } from '../../common/filter/filtersAsync';
+import { log as logger } from '../../logs/logger';
+import { ClientOpts } from '../../common/types/options';
+import { getFilterConfig } from '../config/filters';
+
+export const retrieveAndLoadFilters = async (
+  clientOpts: ClientOpts,
+): Promise<void> => {
+  const globalFilterConfig = getFilterConfig();
+  const filters = await filterRulesLoader(clientOpts.config as CONFIGURATION);
+  globalFilterConfig.loadedFilters = loadAllFilters(filters, clientOpts.config);
+  logger.debug('Loading Filters');
+};

lib/common/filter/filter-rules-loading.ts

@@ -6,6 +6,7 @@ import { log as logger } from '../../logs/logger';
 import { CONFIGURATION, findProjectRoot } from '../config/config';
 import camelcase from 'camelcase';
 import { FiltersType, Rule } from '../types/filter';
+import { retrieveFilters, isValidURI } from './utils';
 
 const SUPPORTED_IAC_EXTENSIONS = ['tf', 'yaml', 'yml', 'tpl', 'json'];
 const IAC_SCM_ORIGINS = [
@@ -139,7 +140,6 @@ function injectRulesAtRuntime(
     config.ACCEPT_GIT;
   if (ACCEPT_CODE) {
     logger.info({ accept: ACCEPT_CODE }, 'Injecting Accept rules for Code/Git');
-
     const templateGET = nestedCopy(
       filters.private.filter(
         (entry) =>
@@ -241,6 +241,7 @@ function injectRulesAtRuntime(
       }
     }
   }
+
   const ACCEPT_APPRISK = process.env.ACCEPT_APPRISK || config.ACCEPT_APPRISK;
   if (ACCEPT_APPRISK) {
     logger.debug(
@@ -318,32 +319,41 @@ function injectRulesAtRuntime(
   return filters;
 }
 
-export default (
+export default async (
   config: CONFIGURATION,
   folderLocation = '',
-): FiltersType | Map<string, FiltersType> => {
+): Promise<FiltersType | Map<string, FiltersType>> => {
   const acceptFilename = config.accept || '';
-  // let filters = config.universalBrokerEnabled
-  //   ? new Map()
-  //   : { private: [], public: [] };
+  let retrievedFilters;
+
+  if (!acceptFilename || (acceptFilename && isValidURI(acceptFilename))) {
+    let rulesUrisMap;
+    if (acceptFilename) {
+      rulesUrisMap = new Map<string, string>().set('current', acceptFilename);
+    } else {
+      rulesUrisMap = new Map<string, string>(
+        Object.entries(config.filterRulesPaths),
+      );
+    }
+    retrievedFilters = await retrieveFilters(rulesUrisMap);
+  }
   let filters;
   if (config.universalBrokerEnabled) {
     filters = new Map();
     const supportedBrokerTypes = config.supportedBrokerTypes;
     supportedBrokerTypes.forEach((type) => {
-      filters[type] = yaml.safeLoad(
-        fs.readFileSync(
-          `${path.resolve(
-            findProjectRoot(__dirname) ?? process.cwd(),
-            `${config.filterRulesPaths[type]}`, // this should handle the override for custom filters
-          )}`,
-          'utf8',
-        ),
-      );
+      if (!retrievedFilters.get(type)) {
+        throw new Error(`Missing filter for ${type}.`);
+      }
+      filters[type] = yaml.safeLoad(retrievedFilters.get(type));
       filters[type] = injectRulesAtRuntime(filters[type], config, type);
     });
   } else {
-    if (acceptFilename) {
+    if (!acceptFilename) {
+      return filters;
+    } else if (acceptFilename && retrievedFilters) {
+      filters = yaml.safeLoad(retrievedFilters.get('current'));
+    } else {
       const acceptLocation = path.resolve(
         folderLocation
           ? folderLocation

lib/common/filter/utils.ts

@@ -1,3 +1,10 @@
+import path from 'node:path';
+import fs from 'fs';
+import { makeSingleRawRequestToDownstream } from '../http/request';
+import { PostFilterPreparedRequest } from '../relay/prepareRequest';
+import version from '../utils/version';
+import { findProjectRoot } from '../config/config';
+
 export const validateHeaders = (headerFilters, requestHeaders = []) => {
   for (const filter of headerFilters) {
     const headerValue = requestHeaders[filter.header];
@@ -13,3 +20,51 @@ export const validateHeaders = (headerFilters, requestHeaders = []) => {
 
   return true;
 };
+
+export const isValidURI = (uri: string) => {
+  try {
+    new URL(uri);
+    return true;
+  } catch (e) {
+    return false;
+  }
+};
+
+/* Retrieve filters from list of uris 
+   Can be uri or local path          */
+export const retrieveFilters = async (locations: Map<string, string>) => {
+  const retrievedFiltersMap = new Map<string, any>();
+  for (const key of locations.keys()) {
+    const location = locations.get(key);
+    if (!location) {
+      throw new Error(`Invalid filter uri for type ${key}`);
+    }
+    if (isValidURI(location)) {
+      const req: PostFilterPreparedRequest = {
+        url: location,
+        headers: { 'user-agent': `Snyk Broker Client ${version}` },
+        method: 'GET',
+      };
+      const filter = await makeSingleRawRequestToDownstream(req);
+      if (filter.statusCode && filter.statusCode > 299) {
+        throw new Error(
+          `Error downloading filter ${key}. Url ${location} returned ${filter.statusCode}`,
+        );
+      }
+      retrievedFiltersMap.set(key, filter.body);
+    } else {
+      retrievedFiltersMap.set(
+        key,
+        fs.readFileSync(
+          `${path.resolve(
+            findProjectRoot(__dirname) ?? process.cwd(),
+            location, // this should handle the override for custom filters
+          )}`,
+          'utf-8',
+        ),
+      );
+    }
+  }
+
+  return retrievedFiltersMap;
+};

lib/common/relay/forwardHttpRequest.ts

@@ -16,6 +16,7 @@ import { streamsStore } from '../http/server-post-stream-handler';
 import { ExtendedLogContext } from '../types/log';
 import { LoadedClientOpts, LoadedServerOpts } from '../types/options';
 import { LOADEDFILTERSET } from '../types/filter';
+import { getFilterConfig } from '../../client/config/filters';
 
 // 1. Request coming in over HTTP conn (logged)
 // 2. Filter for rule match (log and block if no match)
@@ -59,15 +60,17 @@ export const forwardHttpRequest = (
       options.config.brokerType == 'client' &&
       options.config.universalBrokerEnabled
     ) {
-      const clientOptions = options as LoadedClientOpts;
-      const loadedFilters = clientOptions.loadedFilters as Map<
+      const loadedFilters = getFilterConfig().loadedFilters as Map<
         string,
         LOADEDFILTERSET
       >;
       filterResponse =
         loadedFilters
           .get(res.locals.websocket.supportedIntegrationType) // The chosen type is determined by websocket connect middlwr
           ?.public(req) || false;
+    } else if (options.config.brokerType == 'client') {
+      const loadedFilters = getFilterConfig().loadedFilters as LOADEDFILTERSET;
+      filterResponse = loadedFilters.public(req);
     } else {
       const loadedFilters = options.loadedFilters as LOADEDFILTERSET;
       filterResponse = loadedFilters.public(req);

lib/common/relay/forwardHttpRequestOverHttp.ts

@@ -9,6 +9,7 @@ import { makeRequestToDownstream } from '../http/request';
 import { maskToken } from '../utils/token';
 import { LoadedClientOpts, LoadedServerOpts } from '../types/options';
 import { LOADEDFILTERSET } from '../types/filter';
+import { getFilterConfig } from '../../client/config/filters';
 
 // 1. Request coming in over HTTP conn (logged)
 // 2. Filter for rule match (log and block if no match)
@@ -46,15 +47,17 @@ export const forwardHttpRequestOverHttp = (
       options.config.brokerType == 'client' &&
       options.config.universalBrokerEnabled
     ) {
-      const clientOptions = options as LoadedClientOpts;
-      const loadedFilters = clientOptions.loadedFilters as Map<
+      const loadedFilters = getFilterConfig().loadedFilters as Map<
         string,
         LOADEDFILTERSET
       >;
       filterResponse =
         loadedFilters
           .get(res.locals.websocket.supportedIntegrationType) // The chosen type is determined by websocket connect middlwr
           ?.public(req) || false;
+    } else if (options.config.brokerType == 'client') {
+      const loadedFilters = getFilterConfig().loadedFilters as LOADEDFILTERSET;
+      filterResponse = loadedFilters.public(req);
     } else {
       const loadedFilters = options.loadedFilters as LOADEDFILTERSET;
       filterResponse = loadedFilters.public(req);

lib/common/relay/forwardWebsocketRequest.ts

@@ -19,6 +19,7 @@ import { LoadedClientOpts, LoadedServerOpts } from '../types/options';
 import { runPreRequestPlugins } from '../../client/brokerClientPlugins/pluginManager';
 import { computeContentLength } from '../utils/content-length';
 import { contentLengthHeader } from '../utils/headers-value-constants';
+import { getFilterConfig } from '../../client/config/filters';
 
 export const forwardWebSocketRequest = (
   options: LoadedClientOpts | LoadedServerOpts,
@@ -179,20 +180,21 @@ export const forwardWebSocketRequest = (
     );
 
     let filterResponse;
-
     if (
       options.config.brokerType == 'client' &&
       options.config.universalBrokerEnabled
     ) {
-      const clientOptions = options as LoadedClientOpts;
-      const loadedFilters = clientOptions.loadedFilters as Map<
+      const loadedFilters = getFilterConfig().loadedFilters as Map<
         string,
         LOADEDFILTERSET
       >;
       filterResponse =
         loadedFilters
           .get(websocketConnectionHandler.supportedIntegrationType)
           ?.private(payload) || false;
+    } else if (options.config.brokerType == 'client') {
+      const loadedFilters = getFilterConfig().loadedFilters as LOADEDFILTERSET;
+      filterResponse = loadedFilters.private(payload);
     } else {
       const loadedFilters = options.loadedFilters as LOADEDFILTERSET;
       filterResponse = loadedFilters.private(payload);