From 3857332a9e63b58d70c75e419cba0fcd35af6534 Mon Sep 17 00:00:00 2001 From: RahulGautamSingh Date: Wed, 7 Aug 2024 12:58:34 +0530 Subject: [PATCH] feat(self-hosted): convert experimental env vars to config options (#29154) Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> --- docs/usage/self-hosted-configuration.md | 28 +++++++- docs/usage/self-hosted-experimental.md | 19 ------ lib/config/global.ts | 2 + lib/config/options/index.ts | 23 +++++++ lib/config/types.ts | 5 ++ lib/instrumentation/reporting.ts | 2 +- lib/util/s3.spec.ts | 26 ++++++-- lib/util/s3.ts | 14 +++- lib/workers/global/config/parse/env.spec.ts | 7 ++ lib/workers/global/config/parse/env.ts | 3 + lib/workers/global/config/parse/file.spec.ts | 68 +++++++++----------- lib/workers/global/config/parse/file.ts | 5 +- lib/workers/global/config/parse/index.ts | 3 + 13 files changed, 134 insertions(+), 71 deletions(-) diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index 895ea0a3bd774e..f88f4ccc6f8561 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -388,6 +388,16 @@ The above configuration approach will mean the values are redacted in logs like "customEnvVariables": {"SECRET_TOKEN": "{{ secrets.SECRET_TOKEN }}"}, ``` +## deleteConfigFile + +If set to `true` Renovate tries to delete the self-hosted config file after reading it. + +The process that runs Renovate must have the correct permissions to delete the config file. + + +!!! tip + You can tell Renovate where to find your config file with the `RENOVATE_CONFIG_FILE` environment variable. + ## detectGlobalManagerConfig The purpose of this config option is to allow you (as a bot admin) to configure manager-specific files such as a global `.npmrc` file, instead of configuring it in Renovate config. @@ -1082,7 +1092,7 @@ Defines how the report is exposed: - `` If unset, no report will be provided, though the debug logs will still have partial information of the report - `logging` The report will be printed as part of the log messages on `INFO` level - `file` The report will be written to a path provided by [`reportPath`](#reportpath) -- `s3` The report is pushed to an S3 bucket defined by [`reportPath`](#reportpath). This option reuses [`RENOVATE_X_S3_ENDPOINT`](./self-hosted-experimental.md#renovate_x_s3_endpoint) and [`RENOVATE_X_S3_PATH_STYLE`](./self-hosted-experimental.md#renovate_x_s3_path_style) +- `s3` The report is pushed to an S3 bucket defined by [`reportPath`](#reportpath). This option reuses [`s3Endpoint`](./self-hosted-configuration.md#s3endpoint) and [`s3PathStyle`](./self-hosted-configuration.md#s3PathStyle) ## repositories @@ -1140,6 +1150,22 @@ The combinations of `requireConfig` and `onboarding` are: | `requireConfig=optional` | An onboarding PR will be created if no config file exists. If the onboarding PR is closed and there's no config file, the repository will be processed. | Repository is processed regardless of config file presence. | | `requireConfig=ignored` | No onboarding PR will be created and repo will be processed while ignoring any config file present. | Repository is processed, any config file is ignored. | +## s3Endpoint + +If set, Renovate will use this string as the `endpoint` when creating the AWS S3 client instance. + +## s3PathStyle + +If set, Renovate will enable `forcePathStyle` when creating the AWS S3 client instance. + +For example: +| `s3PathStyle` | Path | +| ------------- | ---------------------------------- | +| `off` | `https://bucket.s3.amazonaws.com/` | +| `on` | `https://s3.amazonaws.com/bucket/` | + +Read the [AWS S3 docs, Interface BucketEndpointInputConfig](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/bucketendpointinputconfig.html) to learn more about path-style URLs. + ## secrets Secrets may be configured by a bot admin in `config.js`, which will then make them available for templating within repository configs. diff --git a/docs/usage/self-hosted-experimental.md b/docs/usage/self-hosted-experimental.md index 0bb89f977834ec..8c4ef0524d3ecd 100644 --- a/docs/usage/self-hosted-experimental.md +++ b/docs/usage/self-hosted-experimental.md @@ -28,13 +28,6 @@ Skipping the check will speed things up, but may result in versions being return If set to any value, Renovate will always paginate requests to GitHub fully, instead of stopping after 10 pages. -## `RENOVATE_X_DELETE_CONFIG_FILE` - -If `true` Renovate tries to delete the self-hosted config file after reading it. -You can set the config file Renovate should read with the `RENOVATE_CONFIG_FILE` environment variable. - -The process that runs Renovate must have the correct permissions to delete the config file. - ## `RENOVATE_X_DOCKER_HUB_DISABLE_LABEL_LOOKUP` If set to any value, Renovate will skip attempting to get release labels (e.g. gitRef, sourceUrl) from manifest annotations for `https://index.docker.io`. @@ -113,18 +106,6 @@ If set, Renovate will rewrite GitHub Enterprise Server's pagination responses to If set, Renovate will persist repository cache locally after uploading to S3. -## `RENOVATE_X_S3_ENDPOINT` - -If set, Renovate will use this string as the `endpoint` when instantiating the AWS S3 client. - -## `RENOVATE_X_S3_PATH_STYLE` - -If set, Renovate will enable `forcePathStyle` when instantiating the AWS S3 client. - -> Whether to force path style URLs for S3 objects (e.g., `https://s3.amazonaws.com//` instead of `https://.s3.amazonaws.com/`) - -Source: [AWS S3 documentation - Interface BucketEndpointInputConfig](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/bucketendpointinputconfig.html) - ## `RENOVATE_X_SQLITE_PACKAGE_CACHE` If set, Renovate will use SQLite as the backend for the package cache. diff --git a/lib/config/global.ts b/lib/config/global.ts index 47d959020f044f..51ca2735c5a836 100644 --- a/lib/config/global.ts +++ b/lib/config/global.ts @@ -37,6 +37,8 @@ export class GlobalConfig { 'autodiscoverRepoSort', 'autodiscoverRepoOrder', 'userAgent', + 's3Endpoint', + 's3PathStyle', 'cachePrivatePackages', ]; diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 721a6fc0b47f58..06d18502cfedc6 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2931,6 +2931,29 @@ const options: RenovateOptions[] = [ default: 90, globalOnly: true, }, + { + name: 'deleteConfigFile', + description: + 'If set to `true`, Renovate tries to delete the self-hosted config file after reading it.', + type: 'boolean', + default: false, + globalOnly: true, + }, + { + name: 's3Endpoint', + description: + 'If set, Renovate will use this string as the `endpoint` when creating the AWS S3 client instance.', + type: 'string', + globalOnly: true, + }, + { + name: 's3PathStyle', + description: + 'If set, Renovate will enable `forcePathStyle` when creating the AWS S3 client instance.', + type: 'boolean', + default: false, + globalOnly: true, + }, { name: 'cachePrivatePackages', description: diff --git a/lib/config/types.ts b/lib/config/types.ts index c2e1df6a181760..c211d6e960d9c0 100644 --- a/lib/config/types.ts +++ b/lib/config/types.ts @@ -125,6 +125,7 @@ export interface GlobalOnlyConfig { redisUrl?: string; repositories?: RenovateRepository[]; useCloudMetadataServices?: boolean; + deleteConfigFile?: boolean; } // Config options used within the repository worker, but not user configurable @@ -165,6 +166,8 @@ export interface RepoGlobalConfig { autodiscoverRepoSort?: RepoSortMethod; autodiscoverRepoOrder?: SortMethod; userAgent?: string; + s3Endpoint?: string; + s3PathStyle?: boolean; cachePrivatePackages?: boolean; } @@ -224,6 +227,8 @@ export interface RenovateConfig AssigneesAndReviewersConfig, ConfigMigration, Record { + s3Endpoint?: string; + s3PathStyle?: boolean; reportPath?: string; reportType?: 'logging' | 'file' | 's3' | null; depName?: string; diff --git a/lib/instrumentation/reporting.ts b/lib/instrumentation/reporting.ts index 8b422f3d3d9b41..68952cd5136eca 100644 --- a/lib/instrumentation/reporting.ts +++ b/lib/instrumentation/reporting.ts @@ -89,7 +89,7 @@ export async function exportStats(config: RenovateConfig): Promise { ContentType: 'application/json', }; - const client = getS3Client(); + const client = getS3Client(config.s3Endpoint, config.s3PathStyle); const command = new PutObjectCommand(s3Params); await client.send(command); } diff --git a/lib/util/s3.spec.ts b/lib/util/s3.spec.ts index 9c3b7f3b9ac2e6..d1bb3698243b18 100644 --- a/lib/util/s3.spec.ts +++ b/lib/util/s3.spec.ts @@ -2,8 +2,6 @@ import { getS3Client, parseS3Url } from './s3'; describe('util/s3', () => { afterEach(() => { - delete process.env.RENOVATE_X_S3_ENDPOINT; - delete process.env.RENOVATE_X_S3_PATH_STYLE; jest.resetModules(); }); @@ -28,10 +26,13 @@ describe('util/s3', () => { expect(client1).toBe(client2); }); - it('is uses experimental env', async () => { - process.env.RENOVATE_X_S3_ENDPOINT = 'https://minio.domain.test'; - process.env.RENOVATE_X_S3_PATH_STYLE = 'true'; + it('uses user-configured s3 values', async () => { const s3 = await import('./s3'); + const globalConfig = await import('../config/global'); + globalConfig.GlobalConfig.set({ + s3Endpoint: 'https://minio.domain.test', + s3PathStyle: true, + }); const client1 = s3.getS3Client(); const client2 = getS3Client(); expect(client1).not.toBe(client2); @@ -44,4 +45,19 @@ describe('util/s3', () => { }); expect(client1.config.forcePathStyle).toBeTrue(); }); + + it('uses s3 values from globalConfig instead of GlobalConfig class', async () => { + const s3 = await import('./s3'); + const client1 = s3.getS3Client('https://minio.domain.test', true); + const client2 = getS3Client('https://minio.domain.test', true); + expect(client1).not.toBe(client2); + expect(await client1.config.endpoint?.()).toStrictEqual({ + hostname: 'minio.domain.test', + path: '/', + port: undefined, + protocol: 'https:', + query: undefined, + }); + expect(client1.config.forcePathStyle).toBeTrue(); + }); }); diff --git a/lib/util/s3.ts b/lib/util/s3.ts index 59bfcf6ab31f5d..cb032266add095 100644 --- a/lib/util/s3.ts +++ b/lib/util/s3.ts @@ -1,12 +1,20 @@ // Singleton S3 instance initialized on-demand. import { S3Client } from '@aws-sdk/client-s3'; +import is from '@sindresorhus/is'; +import { GlobalConfig } from '../config/global'; import { parseUrl } from './url'; let s3Instance: S3Client | undefined; -export function getS3Client(): S3Client { +export function getS3Client( + // Only needed if GlobalConfig is not initialized due to some error + s3Endpoint?: string, + s3PathStyle?: boolean, +): S3Client { if (!s3Instance) { - const endpoint = process.env.RENOVATE_X_S3_ENDPOINT; - const forcePathStyle = process.env.RENOVATE_X_S3_PATH_STYLE; + const endpoint = s3Endpoint ?? GlobalConfig.get('s3Endpoint'); + const forcePathStyle = is.undefined(s3PathStyle) + ? !!GlobalConfig.get('s3PathStyle') + : s3PathStyle; s3Instance = new S3Client({ ...(endpoint && { endpoint }), ...(forcePathStyle && { forcePathStyle: true }), diff --git a/lib/workers/global/config/parse/env.spec.ts b/lib/workers/global/config/parse/env.spec.ts index 9bafd1e28b7bba..8cdefe94a41b05 100644 --- a/lib/workers/global/config/parse/env.spec.ts +++ b/lib/workers/global/config/parse/env.spec.ts @@ -274,13 +274,20 @@ describe('workers/global/config/parse/env', () => { RENOVATE_X_AUTODISCOVER_REPO_SORT: 'alpha', RENOVATE_X_DOCKER_MAX_PAGES: '10', RENOVATE_AUTODISCOVER_REPO_ORDER: 'desc', + RENOVATE_X_DELETE_CONFIG_FILE: 'true', + RENOVATE_X_S3_ENDPOINT: 'endpoint', + RENOVATE_X_S3_PATH_STYLE: 'true', }; const config = await env.getConfig(envParam); + expect(config.dockerMaxPages).toBeUndefined(); expect(config).toMatchObject({ mergeConfidenceEndpoint: 'some-url', mergeConfidenceDatasources: ['docker'], autodiscoverRepoSort: 'alpha', autodiscoverRepoOrder: 'desc', + deleteConfigFile: true, + s3Endpoint: 'endpoint', + s3PathStyle: true, }); }); diff --git a/lib/workers/global/config/parse/env.ts b/lib/workers/global/config/parse/env.ts index 349bc5e460b37f..c03393d7c6238a 100644 --- a/lib/workers/global/config/parse/env.ts +++ b/lib/workers/global/config/parse/env.ts @@ -88,6 +88,9 @@ function massageEnvKeyValues(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { const convertedExperimentalEnvVars = [ 'RENOVATE_X_AUTODISCOVER_REPO_SORT', 'RENOVATE_X_AUTODISCOVER_REPO_ORDER', + 'RENOVATE_X_DELETE_CONFIG_FILE', + 'RENOVATE_X_S3_ENDPOINT', + 'RENOVATE_X_S3_PATH_STYLE', 'RENOVATE_X_MERGE_CONFIDENCE_API_BASE_URL', 'RENOVATE_X_MERGE_CONFIDENCE_SUPPORTED_DATASOURCES', ]; diff --git a/lib/workers/global/config/parse/file.spec.ts b/lib/workers/global/config/parse/file.spec.ts index 89f92f183268f1..3df7eede2f3be1 100644 --- a/lib/workers/global/config/parse/file.spec.ts +++ b/lib/workers/global/config/parse/file.spec.ts @@ -141,34 +141,16 @@ describe('workers/global/config/parse/file', () => { expect(logger.fatal).toHaveBeenCalledWith('Unsupported file type'); fs.unlinkSync(configFile); }); - - it('removes the config file if RENOVATE_CONFIG_FILE & RENOVATE_X_DELETE_CONFIG_FILE are set', async () => { - fsRemoveSpy.mockImplementationOnce(() => { - // no-op - }); - fsPathExistsSpy - .mockResolvedValueOnce(true as never) - .mockResolvedValueOnce(true as never); - const configFile = upath.resolve(tmp.path, './config.json'); - fs.writeFileSync(configFile, `{"token": "abc"}`, { encoding: 'utf8' }); - - await file.getConfig({ - RENOVATE_CONFIG_FILE: configFile, - RENOVATE_X_DELETE_CONFIG_FILE: 'true', - }); - - expect(processExitSpy).not.toHaveBeenCalled(); - expect(fsRemoveSpy).toHaveBeenCalledTimes(1); - expect(fsRemoveSpy).toHaveBeenCalledWith(configFile); - fs.unlinkSync(configFile); - }); }); describe('deleteConfigFile()', () => { it.each([[undefined], [' ']])( 'skip when RENOVATE_CONFIG_FILE is not set ("%s")', async (configFile) => { - await file.deleteNonDefaultConfig({ RENOVATE_CONFIG_FILE: configFile }); + await file.deleteNonDefaultConfig( + { RENOVATE_CONFIG_FILE: configFile }, + true, + ); expect(fsRemoveSpy).toHaveBeenCalledTimes(0); }, @@ -177,23 +159,27 @@ describe('workers/global/config/parse/file', () => { it('skip when config file does not exist', async () => { fsPathExistsSpy.mockResolvedValueOnce(false as never); - await file.deleteNonDefaultConfig({ - RENOVATE_CONFIG_FILE: 'path', - RENOVATE_X_DELETE_CONFIG_FILE: 'true', - }); + await file.deleteNonDefaultConfig( + { + RENOVATE_CONFIG_FILE: 'path', + }, + true, + ); expect(fsRemoveSpy).toHaveBeenCalledTimes(0); }); it.each([['false'], [' ']])( - 'skip if RENOVATE_X_DELETE_CONFIG_FILE is not set ("%s")', + 'skip if deleteConfigFile is not set ("%s")', async (deleteConfig) => { fsPathExistsSpy.mockResolvedValueOnce(true as never); - await file.deleteNonDefaultConfig({ - RENOVATE_X_DELETE_CONFIG_FILE: deleteConfig, - RENOVATE_CONFIG_FILE: '/path/to/config.js', - }); + await file.deleteNonDefaultConfig( + { + RENOVATE_CONFIG_FILE: '/path/to/config.js', + }, + deleteConfig === 'true', + ); expect(fsRemoveSpy).toHaveBeenCalledTimes(0); }, @@ -206,10 +192,12 @@ describe('workers/global/config/parse/file', () => { fsPathExistsSpy.mockResolvedValueOnce(true as never); const configFile = '/path/to/config.js'; - await file.deleteNonDefaultConfig({ - RENOVATE_CONFIG_FILE: configFile, - RENOVATE_X_DELETE_CONFIG_FILE: 'true', - }); + await file.deleteNonDefaultConfig( + { + RENOVATE_CONFIG_FILE: configFile, + }, + true, + ); expect(fsRemoveSpy).toHaveBeenCalledTimes(1); expect(fsRemoveSpy).toHaveBeenCalledWith(configFile); @@ -226,10 +214,12 @@ describe('workers/global/config/parse/file', () => { fsPathExistsSpy.mockResolvedValueOnce(true as never); const configFile = '/path/to/config.js'; - await file.deleteNonDefaultConfig({ - RENOVATE_CONFIG_FILE: configFile, - RENOVATE_X_DELETE_CONFIG_FILE: 'true', - }); + await file.deleteNonDefaultConfig( + { + RENOVATE_CONFIG_FILE: configFile, + }, + true, + ); expect(fsRemoveSpy).toHaveBeenCalledTimes(1); expect(fsRemoveSpy).toHaveBeenCalledWith(configFile); diff --git a/lib/workers/global/config/parse/file.ts b/lib/workers/global/config/parse/file.ts index 61657fad2088c1..9d7c815c59ef04 100644 --- a/lib/workers/global/config/parse/file.ts +++ b/lib/workers/global/config/parse/file.ts @@ -79,13 +79,12 @@ export async function getConfig(env: NodeJS.ProcessEnv): Promise { logger.debug('No config file found on disk - skipping'); } - await deleteNonDefaultConfig(env); // Try deletion only if RENOVATE_CONFIG_FILE is specified - return migrateAndValidateConfig(config, configFile); } export async function deleteNonDefaultConfig( env: NodeJS.ProcessEnv, + deleteConfigFile: boolean, ): Promise { const configFile = env.RENOVATE_CONFIG_FILE; @@ -93,7 +92,7 @@ export async function deleteNonDefaultConfig( return; } - if (env.RENOVATE_X_DELETE_CONFIG_FILE !== 'true') { + if (!deleteConfigFile) { return; } diff --git a/lib/workers/global/config/parse/index.ts b/lib/workers/global/config/parse/index.ts index 520321f681558a..b203cbc09e9713 100644 --- a/lib/workers/global/config/parse/index.ts +++ b/lib/workers/global/config/parse/index.ts @@ -98,6 +98,9 @@ export async function parseConfigs( config.forkProcessing = 'enabled'; } + // Only try deletion if RENOVATE_CONFIG_FILE is set + await fileParser.deleteNonDefaultConfig(env, !!config.deleteConfigFile); + // Massage onboardingNoDeps if (!config.autodiscover && config.onboardingNoDeps !== 'disabled') { logger.debug('Enabling onboardingNoDeps while in non-autodiscover mode');