feat: add remote rulesets retrieval support

snyk · Nov 7, 2024 · 54a0956 · 54a0956
1 parent 694a622
commit 54a0956
Show file tree

Hide file tree

Showing 4 changed files with 94 additions and 43 deletions.
diff --git a/config.default.json b/config.default.json
@@ -497,31 +497,31 @@
     }
   },
   "FILTER_RULES_PATHS": {
-    "apprisk": "defaultFilters/apprisk.json",
-    "artifactory": "defaultFilters/artifactory.json",
-    "azure-repos": "defaultFilters/azure-repos.json",
-    "bitbucket-server": "defaultFilters/bitbucket-server.json",
-    "bitbucket-server-bearer-auth": "defaultFilters/bitbucket-server-bearer-auth.json",
-    "github-enterprise": "defaultFilters/github-enterprise.json",
-    "github-server-app": "defaultFilters/github-server-app.json",
-    "github-cloud-app": "defaultFilters/github-cloud-app.json",
-    "github": "defaultFilters/github.json",
-    "gitlab": "defaultFilters/gitlab.json",
-    "jira": "defaultFilters/jira.json",
-    "jira-bearer-auth": "defaultFilters/jira-bearer-auth.json",
-    "nexus": "defaultFilters/nexus-and-nexus2.json",
-    "docker-hub":"defaultFilters/container-registry-agent.json",
-    "ecr":"defaultFilters/container-registry-agent.json",
-    "acr":"defaultFilters/container-registry-agent.json",
-    "gcr":"defaultFilters/container-registry-agent.json",
-    "artifactory-cr":"defaultFilters/container-registry-agent.json",
-    "harbor-cr":"defaultFilters/container-registry-agent.json",
-    "quay-cr":"defaultFilters/container-registry-agent.json",
-    "github-cr":"defaultFilters/container-registry-agent.json",
-    "nexus-cr":"defaultFilters/container-registry-agent.json",
-    "digitalocean-cr":"defaultFilters/container-registry-agent.json",
-    "gitlab-cr":"defaultFilters/container-registry-agent.json",
-    "google-artifact-cr":"defaultFilters/container-registry-agent.json",
-    "workload":"defaultFilters/workload.json"
+    "apprisk": "https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/apprisk.json",
+    "artifactory": "https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/artifactory.json",
+    "azure-repos": "https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/azure-repos.json",
+    "bitbucket-server": "https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/bitbucket-server.json",
+    "bitbucket-server-bearer-auth": "https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/bitbucket-server-bearer-auth.json",
+    "github-enterprise": "https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/github-enterprise.json",
+    "github-server-app": "https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/github-server-app.json",
+    "github-cloud-app": "https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/github-cloud-app.json",
+    "github": "https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/github.json",
+    "gitlab": "https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/gitlab.json",
+    "jira": "https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/jira.json",
+    "jira-bearer-auth": "https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/jira-bearer-auth.json",
+    "nexus": "https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/nexus-and-nexus2.json",
+    "docker-hub":"https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/container-registry-agent.json",
+    "ecr":"https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/container-registry-agent.json",
+    "acr":"https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/container-registry-agent.json",
+    "gcr":"https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/container-registry-agent.json",
+    "artifactory-cr":"https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/container-registry-agent.json",
+    "harbor-cr":"https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/container-registry-agent.json",
+    "quay-cr":"https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/container-registry-agent.json",
+    "github-cr":"https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/container-registry-agent.json",
+    "nexus-cr":"https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/container-registry-agent.json",
+    "digitalocean-cr":"https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/container-registry-agent.json",
+    "gitlab-cr":"https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/container-registry-agent.json",
+    "google-artifact-cr":"https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/container-registry-agent.json",
+    "workload":"https://raw.githubusercontent.com/snyk/broker/refs/heads/master/defaultFilters/workload.json"
   }
 }
diff --git a/lib/common/filter/filter-rules-loading.ts b/lib/common/filter/filter-rules-loading.ts
@@ -6,6 +6,7 @@ import { log as logger } from '../../logs/logger';
 import { CONFIGURATION, findProjectRoot } from '../config/config';
 import camelcase from 'camelcase';
 import { FiltersType, Rule } from '../types/filter';
+import { retrieveFilters, isValidURI } from './utils';
 
 const SUPPORTED_IAC_EXTENSIONS = ['tf', 'yaml', 'yml', 'tpl', 'json'];
 const IAC_SCM_ORIGINS = [
@@ -318,32 +319,37 @@ function injectRulesAtRuntime(
   return filters;
 }
 
-export default (
+export default async (
   config: CONFIGURATION,
   folderLocation = '',
-): FiltersType | Map<string, FiltersType> => {
+): Promise<FiltersType | Map<string, FiltersType>> => {
   const acceptFilename = config.accept || '';
-  // let filters = config.universalBrokerEnabled
-  //   ? new Map()
-  //   : { private: [], public: [] };
+  let retrievedFilters;
+
+  if (!acceptFilename || (acceptFilename && isValidURI(acceptFilename))) {
+    let rulesUrisMap;
+    if (acceptFilename) {
+      rulesUrisMap = new Map<string, string>().set('current', acceptFilename);
+    } else {
+      rulesUrisMap = new Map<string, string>(
+        Object.entries(config.filterRulesPaths),
+      );
+    }
+    retrievedFilters = await retrieveFilters(rulesUrisMap);
+  }
+
   let filters;
   if (config.universalBrokerEnabled) {
     filters = new Map();
     const supportedBrokerTypes = config.supportedBrokerTypes;
     supportedBrokerTypes.forEach((type) => {
-      filters[type] = yaml.safeLoad(
-        fs.readFileSync(
-          `${path.resolve(
-            findProjectRoot(__dirname) ?? process.cwd(),
-            `${config.filterRulesPaths[type]}`, // this should handle the override for custom filters
-          )}`,
-          'utf8',
-        ),
-      );
+      filters[type] = yaml.safeLoad(retrievedFilters[type]);
       filters[type] = injectRulesAtRuntime(filters[type], config, type);
     });
   } else {
-    if (acceptFilename) {
+    if (acceptFilename && retrievedFilters) {
+      filters = yaml.safeLoad(retrievedFilters.get('current'));
+    } else {
       const acceptLocation = path.resolve(
         folderLocation
           ? folderLocation

diff --git a/lib/common/filter/utils.ts b/lib/common/filter/utils.ts
@@ -1,3 +1,9 @@
+import path from 'node:path';
+import { makeSingleRawRequestToDownstream } from '../http/request';
+import { PostFilterPreparedRequest } from '../relay/prepareRequest';
+import version from '../utils/version';
+import { findProjectRoot } from '../config/config';
+
 export const validateHeaders = (headerFilters, requestHeaders = []) => {
   for (const filter of headerFilters) {
     const headerValue = requestHeaders[filter.header];
@@ -13,3 +19,43 @@ export const validateHeaders = (headerFilters, requestHeaders = []) => {
 
   return true;
 };
+
+export const isValidURI = (uri: string) => {
+  try {
+    new URL(uri);
+    return true;
+  } catch (e) {
+    return false;
+  }
+};
+
+/* Retrieve filters from list of uris 
+   Can be uri or local path          */
+export const retrieveFilters = async (locations: Map<string, string>) => {
+  const retrievedFiltersMap = new Map<string, any>();
+  for (const key of locations.keys()) {
+    const location = locations.get(key);
+    if (!location) {
+      throw new Error(`Invalid filter uri for type ${key}`);
+    }
+    if (isValidURI(location)) {
+      const req: PostFilterPreparedRequest = {
+        url: location,
+        headers: { 'user-agent': `Snyk Broker Client ${version}` },
+        method: 'GET',
+      };
+      const filter = await makeSingleRawRequestToDownstream(req);
+      retrievedFiltersMap.set(key, filter.body);
+    } else {
+      retrievedFiltersMap.set(
+        key,
+        `${path.resolve(
+          findProjectRoot(__dirname) ?? process.cwd(),
+          location, // this should handle the override for custom filters
+        )}`,
+      );
+    }
+  }
+
+  return retrievedFiltersMap;
+};
diff --git a/lib/index.ts b/lib/index.ts
@@ -46,7 +46,7 @@ export const app = async ({ port = 7341, client = false, config }) => {
       any
     > as CONFIGURATION;
     localConfig.brokerType = method;
-    const filters = filterRulesLoader(localConfig);
+    const filters = await filterRulesLoader(localConfig);
     if (!filters) {
       const error = new ReferenceError(
         `No Filters found. A Broker requires filters to run. Shutting down.`,
@@ -56,7 +56,6 @@ export const app = async ({ port = 7341, client = false, config }) => {
       throw error;
     } else {
       if (method == 'client') {
-        // if the localConfig has the broker server, then we must assume it's a client
         return await (
           await import('./client')
         ).main({