From 005111187912a56e587dd849f1ff5f7ba9c46480 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Sat, 20 May 2023 22:36:59 +0200 Subject: [PATCH] Leverage t3-oss/env-core for better env vars --- .github/workflows/validate-workflow.yml | 2 +- apps/app-config/_generateDocs.bin.ts | 8 +- apps/app-config/backendConfig.ts | 6 +- apps/app-config/bootstrap.ts | 2 +- apps/app-config/commonConfig.ts | 31 +---- apps/app-config/env.ts | 166 +++++++++--------------- apps/app-config/integration-envs.ts | 68 ++++++++++ apps/app-config/package.json | 3 +- apps/cli/_cli.ts | 15 ++- apps/web/contexts/atoms.tsx | 8 -- pnpm-lock.yaml | 24 ++++ 11 files changed, 174 insertions(+), 159 deletions(-) create mode 100644 apps/app-config/integration-envs.ts diff --git a/.github/workflows/validate-workflow.yml b/.github/workflows/validate-workflow.yml index d35204fc..2ef9437e 100644 --- a/.github/workflows/validate-workflow.yml +++ b/.github/workflows/validate-workflow.yml @@ -73,7 +73,7 @@ jobs: # TODO: Figure out a pattern to make environment variables parsed / required on demand rather than on startup time # Ideally have a way to switch between the two... Where we can also choose proactive parsing for sanity checking... - name: Run health check - run: JWT_SECRET_OR_PUBLIC_KEY=NOOP POSTGRES_OR_WEBHOOK_URL=noop NEXT_PUBLIC_SUPABASE_URL=noop NEXT_PUBLIC_SUPABASE_ANON_KEY=noop NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=noop CLERK_SECRET_KEY=noop node --loader tsx ./bin/venice.ts health + run: SKIP_ENV_VALIDATION=1 JWT_SECRET_OR_PUBLIC_KEY=NOOP POSTGRES_OR_WEBHOOK_URL=noop node --loader tsx ./bin/venice.ts health - name: Run migration check run: POSTGRES_OR_WEBHOOK_URL=postgres://postgres:test@localhost:5432/test pnpm migration up diff --git a/apps/app-config/_generateDocs.bin.ts b/apps/app-config/_generateDocs.bin.ts index 32b2944f..64879ded 100644 --- a/apps/app-config/_generateDocs.bin.ts +++ b/apps/app-config/_generateDocs.bin.ts @@ -3,13 +3,14 @@ import * as path from 'node:path' import tablemark from 'tablemark' -import {buildUrl, R, zParser} from '@usevenice/util' +import {buildUrl, R} from '@usevenice/util' -import {parseIntConfigsFromRawEnv, zAllEnv} from './env' +import {env, envConfig} from './env' +import {parseIntConfigsFromRawEnv} from './integration-envs' import {DOCUMENTED_PROVIDERS} from './providers' const envList = R.pipe( - zAllEnv.shape, + {...envConfig.server, ...envConfig.client}, R.toPairs, R.filter( ([key]) => @@ -124,7 +125,6 @@ console.log(`Wrote ${dotEnvExampleOutPath}`) if (process.env.NODE_ENV !== 'production') { console.log('Test out loading env vars') - const env = zParser(zAllEnv).parseUnknown(process.env) const configs = parseIntConfigsFromRawEnv() console.log('Parsed env', env) console.log('Parsed intConfigs', configs) diff --git a/apps/app-config/backendConfig.ts b/apps/app-config/backendConfig.ts index 67fe8f19..7667fb77 100644 --- a/apps/app-config/backendConfig.ts +++ b/apps/app-config/backendConfig.ts @@ -10,19 +10,17 @@ import { import {makePostgresMetaService} from '@usevenice/core-integration-postgres' import type {PipelineInput} from '@usevenice/engine-backend' import {getContextFactory} from '@usevenice/engine-backend' -import {joinPath, R, Rx, zParser} from '@usevenice/util' +import {joinPath, R, Rx} from '@usevenice/util' import {veniceCommonConfig} from './commonConfig' import {getServerUrl} from './constants' -import {zAllEnv} from './env' +import {env} from './env' import type {PROVIDERS} from './providers' export {DatabaseError} from '@usevenice/core-integration-postgres/register.node' export {Papa} from '@usevenice/integration-import' export {makePostgresClient} from '@usevenice/integration-postgres' -const env = zParser(zAllEnv).parseUnknown(process.env) - export const backendEnv = env const usePg = env.POSTGRES_OR_WEBHOOK_URL.startsWith('postgres') diff --git a/apps/app-config/bootstrap.ts b/apps/app-config/bootstrap.ts index a97e5718..73923c30 100644 --- a/apps/app-config/bootstrap.ts +++ b/apps/app-config/bootstrap.ts @@ -6,7 +6,7 @@ import {getEnvVar} from '@usevenice/util' import {contextFactory} from './backendConfig' import type {PROVIDERS} from './providers' -import {parseIntConfigsFromRawEnv} from './env' +import {parseIntConfigsFromRawEnv} from './integration-envs' export type _ResourceInput = IntegrationInput<(typeof PROVIDERS)[number]> diff --git a/apps/app-config/commonConfig.ts b/apps/app-config/commonConfig.ts index 62d74586..b3ba55f0 100644 --- a/apps/app-config/commonConfig.ts +++ b/apps/app-config/commonConfig.ts @@ -1,37 +1,14 @@ -import type {AnySyncProvider, EnvName, LinkFactory} from '@usevenice/cdk-core' +import type {AnySyncProvider, LinkFactory} from '@usevenice/cdk-core' import type {SyncEngineCommonConfig} from '@usevenice/engine-frontend' -import {joinPath, zParser} from '@usevenice/util' +import {joinPath} from '@usevenice/util' import {getServerUrl} from './constants' -import {zCommonEnv} from './env' +import {env} from './env' import {PROVIDERS} from './providers' export {Papa} from '@usevenice/integration-import' -type VercelEnv = 'production' | 'preview' | 'development' - -/* eslint-disable @typescript-eslint/no-non-null-assertion */ -export const commonEnv = zParser(zCommonEnv).parse({ - // Need to use fully qualified form of process.env.$VAR for - // webpack DefineEnv that next.js uses to work - // TODO: Maybe we define defaults inside env.ts to avoid duplicating ourselves? - NEXT_PUBLIC_SUPABASE_URL: process.env['NEXT_PUBLIC_SUPABASE_URL']!, - NEXT_PUBLIC_SUPABASE_ANON_KEY: process.env['NEXT_PUBLIC_SUPABASE_ANON_KEY']!, - NEXT_PUBLIC_SENTRY_DSN: process.env['NEXT_PUBLIC_SENTRY_DSN']!, - NEXT_PUBLIC_POSTHOG_WRITEKEY: process.env['NEXT_PUBLIC_POSTHOG_WRITEKEY']!, - NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: - process.env['NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY']!, - NEXT_PUBLIC_CLERK_SUPABASE_JWT_TEMPLATE_NAME: - process.env['NEXT_PUBLIC_CLERK_SUPABASE_JWT_TEMPLATE_NAME']!, - DEFAULT_CONNECT_ENV: ( - { - production: 'production', - preview: 'development', - development: 'sandbox', - } satisfies Record - )[process.env['NEXT_PUBLIC_VERCEL_ENV'] ?? ''], -}) -/* eslint-enable @typescript-eslint/no-non-null-assertion */ +export const commonEnv = env // TODO: Removing providers we are not using so we don't have nearly as much code, at least on the frontend! // Further perhaps code from supported providers can be loaded dynamically based on diff --git a/apps/app-config/env.ts b/apps/app-config/env.ts index 15d17154..b395a035 100644 --- a/apps/app-config/env.ts +++ b/apps/app-config/env.ts @@ -1,107 +1,59 @@ -import {makeId, zEnvName} from '@usevenice/cdk-core' -import {R, z, zEnvVars, zFlattenForEnv} from '@usevenice/util' - -import type {PROVIDERS} from './providers' -import {DOCUMENTED_PROVIDERS} from './providers' - -// MARK: - Env vars - -export const zCommonEnv = zEnvVars({ - NEXT_PUBLIC_SUPABASE_URL: z.string(), - NEXT_PUBLIC_SUPABASE_ANON_KEY: z.string(), - NEXT_PUBLIC_SENTRY_DSN: z.string().optional(), - NEXT_PUBLIC_POSTHOG_WRITEKEY: z.string().optional(), - NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: z.string(), - NEXT_PUBLIC_CLERK_SUPABASE_JWT_TEMPLATE_NAME: z.string().default('supabase'), - - // Deprecated - // TODO: Deprecate me? prefix with NEXT_PUBLIC please - DEFAULT_CONNECT_ENV: zEnvName.default('sandbox'), - - // TODO: Make use of me... prefix with NEXT_PUBLIC please - NODE_ENV: z - .string() - .optional() - .default(process.env['NEXT_PUBLIC_NODE_ENV'] ?? process.env.NODE_ENV), -}) - -export const zBackendEnv = zEnvVars({ - POSTGRES_OR_WEBHOOK_URL: z.string().describe(` - Pass a valid postgres(ql):// url for stateful mode. Will be used Primary database used for metadata and user data storage - Pass a valid http(s):// url for stateless mode. Sync data and metadata be sent to provided URL and you are responsible for your own persistence`), - JWT_SECRET_OR_PUBLIC_KEY: z - .string() - .trim() - .describe('Used for validating authenticity of accessToken'), - - CLERK_SECRET_KEY: z.string(), - - SENTRY_CRON_MONITOR_ID: z - .string() - .optional() - .describe('Used to monitor the schedule syncs cron job'), -}) - -/** We would prefer to use `.` but vercel env var name can only be number, letter and underscore... */ -const separator = '__' -const getPrefix = (name: string) => makeId('int', name, '') - -// Should this be all providers or only dcoumented ones? - -export const zFlatConfigByProvider = R.mapToObj(DOCUMENTED_PROVIDERS, (p) => [ - p.name, - zFlattenForEnv(p.def.integrationConfig ?? z.unknown(), { - prefix: getPrefix(p.name), - separator, - }), -]) - -export const zIntegrationEnv = zEnvVars( - R.pipe( - zFlatConfigByProvider, - R.values, - R.map((schema) => schema.innerType().shape), - R.mergeAll, - ) as {}, -) - -export const zAllEnv = zCommonEnv.merge(zBackendEnv).merge(zIntegrationEnv) - -// MARK: - Parsing integration configs - -/** - * Input env must be raw, so means most likely we are parsing the flatConfig input twice - * for the moment unfortunately... But we need this to support transforms in flatConfig - */ -export function parseIntConfigsFromRawEnv( - env: Record = process.env, -) { - return R.pipe( - R.mapValues(zFlatConfigByProvider, (zFlatConfig, name) => { - const subEnv = R.pipe( - R.pickBy(env, (_v, k) => k.startsWith(getPrefix(name))), - (e) => (R.keys(e).length ? e : undefined), // To get .optional() to work - ) - try { - return zFlatConfig.optional().parse(subEnv) - } catch (err) { - if (err instanceof z.ZodError && err.issues[0]) { - const issue = err.issues[0] - // const msg = issue.code === 'invalid_type' && issue.message === 'Required' ? `` - // console.log('subEnv', subEnv, issue) - throw new Error( - `Failed to configure "${name}" provider due to invalid env var "${issue.path.join( - separator, - )}": ${issue.message} [${issue.code}]`, - ) - } - } - }), - (configMap) => R.pickBy(configMap, (val) => val !== undefined), - ) as { - [k in (typeof PROVIDERS)[number]['name']]?: Extract< - (typeof PROVIDERS)[number], - {name: k} - >['def']['_types']['integrationConfig'] - } -} +import {createEnv} from '@t3-oss/env-nextjs' +import {z} from 'zod' + +export const envConfig = { + server: { + POSTGRES_OR_WEBHOOK_URL: z.string().describe(` +Pass a valid postgres(ql):// url for stateful mode. Will be used Primary database used for metadata and user data storage +Pass a valid http(s):// url for stateless mode. Sync data and metadata be sent to provided URL and you are responsible for your own persistence`), + JWT_SECRET_OR_PUBLIC_KEY: z + .string() + .trim() + .describe('Used for validating authenticity of accessToken'), + + CLERK_SECRET_KEY: z.string(), + SENTRY_CRON_MONITOR_ID: z + .string() + .optional() + .describe('Used to monitor the schedule syncs cron job'), + + INNGEST_EVENT_KEY: z.string(), + INNGEST_SIGNING_KEY: z.string(), + }, + client: { + NEXT_PUBLIC_SUPABASE_URL: z.string(), + NEXT_PUBLIC_SUPABASE_ANON_KEY: z.string(), + NEXT_PUBLIC_SENTRY_DSN: z.string().optional(), + NEXT_PUBLIC_SENTRY_ORG: z.string().optional(), + NEXT_PUBLIC_POSTHOG_WRITEKEY: z.string().optional(), + NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: z.string(), + NEXT_PUBLIC_CLERK_SUPABASE_JWT_TEMPLATE_NAME: z + .string() + .default('supabase'), + }, + runtimeEnv: { + CLERK_SECRET_KEY: process.env['CLERK_SECRET_KEY'], + JWT_SECRET_OR_PUBLIC_KEY: process.env['JWT_SECRET_OR_PUBLIC_KEY'], + POSTGRES_OR_WEBHOOK_URL: process.env['POSTGRES_OR_WEBHOOK_URL'], + SENTRY_CRON_MONITOR_ID: process.env['SENTRY_CRON_MONITOR_ID'], + NEXT_PUBLIC_SENTRY_ORG: process.env['NEXT_PUBLIC_SENTRY_ORG'], + NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: + process.env['NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY'], + NEXT_PUBLIC_CLERK_SUPABASE_JWT_TEMPLATE_NAME: + process.env['NEXT_PUBLIC_CLERK_SUPABASE_JWT_TEMPLATE_NAME'], + NEXT_PUBLIC_POSTHOG_WRITEKEY: process.env['NEXT_PUBLIC_POSTHOG_WRITEKEY'], + NEXT_PUBLIC_SENTRY_DSN: process.env['NEXT_PUBLIC_SENTRY_DSN'], + NEXT_PUBLIC_SUPABASE_ANON_KEY: process.env['NEXT_PUBLIC_SUPABASE_ANON_KEY'], + NEXT_PUBLIC_SUPABASE_URL: process.env['NEXT_PUBLIC_SUPABASE_URL'], + INNGEST_EVENT_KEY: process.env['INNGEST_EVENT_KEY'], + INNGEST_SIGNING_KEY: process.env['INNGEST_SIGNING_KEY'], + }, + onInvalidAccess: (variable: string) => { + throw new Error( + `❌ Attempted to access server-side environment variable ${variable} on the client`, + ) + }, + skipValidation: !!process.env['SKIP_ENV_VALIDATION'], +} satisfies Parameters[0] + +export const env = createEnv(envConfig) diff --git a/apps/app-config/integration-envs.ts b/apps/app-config/integration-envs.ts new file mode 100644 index 00000000..897a24f0 --- /dev/null +++ b/apps/app-config/integration-envs.ts @@ -0,0 +1,68 @@ +/** @deprecated. We no longer initialize integration from ENVs, but maybe in clis still? */ +import {makeId} from '@usevenice/cdk-core' +import {R, z, zEnvVars, zFlattenForEnv} from '@usevenice/util' + +import type {PROVIDERS} from './providers' +import {DOCUMENTED_PROVIDERS} from './providers' + +/** We would prefer to use `.` but vercel env var name can only be number, letter and underscore... */ +const separator = '__' +const getPrefix = (name: string) => makeId('int', name, '') + +// Should this be all providers or only dcoumented ones? + +export const zFlatConfigByProvider = R.mapToObj(DOCUMENTED_PROVIDERS, (p) => [ + p.name, + zFlattenForEnv(p.def.integrationConfig ?? z.unknown(), { + prefix: getPrefix(p.name), + separator, + }), +]) + +export const zIntegrationEnv = zEnvVars( + R.pipe( + zFlatConfigByProvider, + R.values, + R.map((schema) => schema.innerType().shape), + R.mergeAll, + ) as {}, +) + +// MARK: - Parsing integration configs + +/** + * Input env must be raw, so means most likely we are parsing the flatConfig input twice + * for the moment unfortunately... But we need this to support transforms in flatConfig + */ +export function parseIntConfigsFromRawEnv( + env: Record = process.env, +) { + return R.pipe( + R.mapValues(zFlatConfigByProvider, (zFlatConfig, name) => { + const subEnv = R.pipe( + R.pickBy(env, (_v, k) => k.startsWith(getPrefix(name))), + (e) => (R.keys(e).length ? e : undefined), // To get .optional() to work + ) + try { + return zFlatConfig.optional().parse(subEnv) + } catch (err) { + if (err instanceof z.ZodError && err.issues[0]) { + const issue = err.issues[0] + // const msg = issue.code === 'invalid_type' && issue.message === 'Required' ? `` + // console.log('subEnv', subEnv, issue) + throw new Error( + `Failed to configure "${name}" provider due to invalid env var "${issue.path.join( + separator, + )}": ${issue.message} [${issue.code}]`, + ) + } + } + }), + (configMap) => R.pickBy(configMap, (val) => val !== undefined), + ) as { + [k in (typeof PROVIDERS)[number]['name']]?: Extract< + (typeof PROVIDERS)[number], + {name: k} + >['def']['_types']['integrationConfig'] + } +} diff --git a/apps/app-config/package.json b/apps/app-config/package.json index be8e6e9b..f54415fe 100644 --- a/apps/app-config/package.json +++ b/apps/app-config/package.json @@ -9,6 +9,7 @@ "dev": "pnpm tsx watch ./_generateDocs.bin.ts" }, "dependencies": { + "@t3-oss/env-nextjs": "0.3.1", "@usevenice/cdk-core": "workspace:*", "@usevenice/cdk-ledger": "workspace:*", "@usevenice/core-integration-airtable": "workspace:*", @@ -22,6 +23,7 @@ "@usevenice/engine-frontend": "workspace:*", "@usevenice/integration-alphavantage": "workspace:*", "@usevenice/integration-beancount": "workspace:*", + "@usevenice/integration-brex": "workspace:*", "@usevenice/integration-expensify": "workspace:*", "@usevenice/integration-foreceipt": "workspace:*", "@usevenice/integration-heron": "workspace:*", @@ -41,7 +43,6 @@ "@usevenice/integration-toggl": "workspace:*", "@usevenice/integration-venmo": "workspace:*", "@usevenice/integration-wise": "workspace:*", - "@usevenice/integration-brex": "workspace:*", "@usevenice/integration-yodlee": "workspace:*", "@usevenice/util": "workspace:*", "chokidar": "3.5.3", diff --git a/apps/cli/_cli.ts b/apps/cli/_cli.ts index 5a6bc705..4d057452 100644 --- a/apps/cli/_cli.ts +++ b/apps/cli/_cli.ts @@ -1,15 +1,17 @@ /* eslint-disable @typescript-eslint/no-unsafe-argument */ import '@usevenice/app-config/register.node' +import {parseIntConfigsFromRawEnv} from '@usevenice/app-config/integration-envs' import type {PROVIDERS} from '@usevenice/app-config/providers' -import {parseIntConfigsFromRawEnv, zAllEnv} from '@usevenice/app-config/env' +import {makeJwtClient} from '@usevenice/cdk-core' import { makePostgresClient, makePostgresMetaService, } from '@usevenice/core-integration-postgres' -import {makeJwtClient} from '@usevenice/cdk-core' import {makeAlphavantageClient} from '@usevenice/integration-alphavantage' +import {makeHeronClient} from '@usevenice/integration-heron' import {makeLunchmoneyClient} from '@usevenice/integration-lunchmoney' +import {makeMergeClient} from '@usevenice/integration-merge' import {makeMootaClient} from '@usevenice/integration-moota' import {makeOneBrickClient} from '@usevenice/integration-onebrick' // Make this import dynamic at runtime, so we can do @@ -26,19 +28,20 @@ import {makeTogglClient} from '@usevenice/integration-toggl' import {makeWiseClient} from '@usevenice/integration-wise' import {makeYodleeClient} from '@usevenice/integration-yodlee' import type {ZFunctionMap} from '@usevenice/util' -import {getEnvVar, R, z, zodInsecureDebug, zParser} from '@usevenice/util' +import {getEnvVar, R, z, zodInsecureDebug} from '@usevenice/util' import type {CliOpts} from './cli-utils' import {cliFromZFunctionMap} from './cli-utils' -import {makeMergeClient} from '@usevenice/integration-merge' -import {makeHeronClient} from '@usevenice/integration-heron' if (getEnvVar('DEBUG_ZOD')) { zodInsecureDebug() } function env() { - return zParser(zAllEnv).parseUnknown(process.env) + process.env['SKIP_ENV_VALIDATION'] = 'true' + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + return require('@usevenice/app-config/env') + .env as typeof import('@usevenice/app-config/env')['env'] } function intConfig(name: T) { diff --git a/apps/web/contexts/atoms.tsx b/apps/web/contexts/atoms.tsx index dda18d32..301ed546 100644 --- a/apps/web/contexts/atoms.tsx +++ b/apps/web/contexts/atoms.tsx @@ -3,20 +3,12 @@ import {useRouter} from 'next/router' import {BooleanParam, createEnumParam, StringParam} from 'use-query-params' -import type {EnvName} from '@usevenice/cdk-core' -import {zEnvName} from '@usevenice/cdk-core' import {parseQueryParams, shallowOmitUndefined} from '@usevenice/util' -import {commonEnv} from '@usevenice/app-config/commonConfig' import {kAccessToken, kEnv} from '../lib/constants' import {atomWithQueryParam} from './utils/atomWithQueryParam' export const accessTokenAtom = atomWithQueryParam(kAccessToken, '', StringParam) -export const envAtom = atomWithQueryParam( - kEnv, - commonEnv.DEFAULT_CONNECT_ENV, - createEnumParam(zEnvName.options), -) export const developerModeAtom = atomWithQueryParam( 'developerMode', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4a1a8c22..4b1cee8f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -145,6 +145,9 @@ importers: apps/app-config: dependencies: + '@t3-oss/env-nextjs': + specifier: 0.3.1 + version: 0.3.1(typescript@5.0.4)(zod@3.21.4) '@usevenice/cdk-core': specifier: workspace:* version: link:../../packages/cdk-core @@ -6490,6 +6493,27 @@ packages: dependencies: defer-to-connect: 2.0.1 + /@t3-oss/env-core@0.3.1(typescript@5.0.4)(zod@3.21.4): + resolution: {integrity: sha512-iEnBuWeSjzqQLDTUw7H+YhstV4OZrGXTkQGL6ZOMxZQoCmwGX7GVS+1KCd5RvCzOtrIAD9jeOItSWNjC7sG4Sg==} + peerDependencies: + typescript: '>=4.7.2' + zod: ^3.0.0 + dependencies: + typescript: 5.0.4 + zod: 3.21.4(patch_hash=bzwjzhue3hmpww5lnv24u5k2ru) + dev: false + + /@t3-oss/env-nextjs@0.3.1(typescript@5.0.4)(zod@3.21.4): + resolution: {integrity: sha512-W1OgOn5xtpdEGraAQesyLzO2aNLRfSJEyK6qjQFfEUnrPbkvB+WxABX2bPMqfn4KJQ8pziLCSdBFiUN8OagqAg==} + peerDependencies: + typescript: '>=4.7.2' + zod: ^3.0.0 + dependencies: + '@t3-oss/env-core': 0.3.1(typescript@5.0.4)(zod@3.21.4) + typescript: 5.0.4 + zod: 3.21.4(patch_hash=bzwjzhue3hmpww5lnv24u5k2ru) + dev: false + /@tanstack/query-core@4.27.0: resolution: {integrity: sha512-sm+QncWaPmM73IPwFlmWSKPqjdTXZeFf/7aEmWh00z7yl2FjqophPt0dE1EHW9P1giMC5rMviv7OUbSDmWzXXA==}