Skip to content

Commit

Permalink
Leverage t3-oss/env-core for better env vars
Browse files Browse the repository at this point in the history
  • Loading branch information
tonyxiao committed May 20, 2023
1 parent 8c026be commit 0051111
Show file tree
Hide file tree
Showing 11 changed files with 174 additions and 159 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/validate-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ jobs:
# TODO: Figure out a pattern to make environment variables parsed / required on demand rather than on startup time
# Ideally have a way to switch between the two... Where we can also choose proactive parsing for sanity checking...
- name: Run health check
run: JWT_SECRET_OR_PUBLIC_KEY=NOOP POSTGRES_OR_WEBHOOK_URL=noop NEXT_PUBLIC_SUPABASE_URL=noop NEXT_PUBLIC_SUPABASE_ANON_KEY=noop NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=noop CLERK_SECRET_KEY=noop node --loader tsx ./bin/venice.ts health
run: SKIP_ENV_VALIDATION=1 JWT_SECRET_OR_PUBLIC_KEY=NOOP POSTGRES_OR_WEBHOOK_URL=noop node --loader tsx ./bin/venice.ts health

- name: Run migration check
run: POSTGRES_OR_WEBHOOK_URL=postgres://postgres:test@localhost:5432/test pnpm migration up
Expand Down
8 changes: 4 additions & 4 deletions apps/app-config/_generateDocs.bin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@ import * as path from 'node:path'

import tablemark from 'tablemark'

import {buildUrl, R, zParser} from '@usevenice/util'
import {buildUrl, R} from '@usevenice/util'

import {parseIntConfigsFromRawEnv, zAllEnv} from './env'
import {env, envConfig} from './env'
import {parseIntConfigsFromRawEnv} from './integration-envs'
import {DOCUMENTED_PROVIDERS} from './providers'

const envList = R.pipe(
zAllEnv.shape,
{...envConfig.server, ...envConfig.client},
R.toPairs,
R.filter(
([key]) =>
Expand Down Expand Up @@ -124,7 +125,6 @@ console.log(`Wrote ${dotEnvExampleOutPath}`)

if (process.env.NODE_ENV !== 'production') {
console.log('Test out loading env vars')
const env = zParser(zAllEnv).parseUnknown(process.env)
const configs = parseIntConfigsFromRawEnv()
console.log('Parsed env', env)
console.log('Parsed intConfigs', configs)
Expand Down
6 changes: 2 additions & 4 deletions apps/app-config/backendConfig.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,17 @@ import {
import {makePostgresMetaService} from '@usevenice/core-integration-postgres'
import type {PipelineInput} from '@usevenice/engine-backend'
import {getContextFactory} from '@usevenice/engine-backend'
import {joinPath, R, Rx, zParser} from '@usevenice/util'
import {joinPath, R, Rx} from '@usevenice/util'

import {veniceCommonConfig} from './commonConfig'
import {getServerUrl} from './constants'
import {zAllEnv} from './env'
import {env} from './env'
import type {PROVIDERS} from './providers'

export {DatabaseError} from '@usevenice/core-integration-postgres/register.node'
export {Papa} from '@usevenice/integration-import'
export {makePostgresClient} from '@usevenice/integration-postgres'

const env = zParser(zAllEnv).parseUnknown(process.env)

export const backendEnv = env

const usePg = env.POSTGRES_OR_WEBHOOK_URL.startsWith('postgres')
Expand Down
2 changes: 1 addition & 1 deletion apps/app-config/bootstrap.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import {getEnvVar} from '@usevenice/util'

import {contextFactory} from './backendConfig'
import type {PROVIDERS} from './providers'
import {parseIntConfigsFromRawEnv} from './env'
import {parseIntConfigsFromRawEnv} from './integration-envs'

export type _ResourceInput = IntegrationInput<(typeof PROVIDERS)[number]>

Expand Down
31 changes: 4 additions & 27 deletions apps/app-config/commonConfig.ts
Original file line number Diff line number Diff line change
@@ -1,37 +1,14 @@
import type {AnySyncProvider, EnvName, LinkFactory} from '@usevenice/cdk-core'
import type {AnySyncProvider, LinkFactory} from '@usevenice/cdk-core'
import type {SyncEngineCommonConfig} from '@usevenice/engine-frontend'
import {joinPath, zParser} from '@usevenice/util'
import {joinPath} from '@usevenice/util'

import {getServerUrl} from './constants'
import {zCommonEnv} from './env'
import {env} from './env'
import {PROVIDERS} from './providers'

export {Papa} from '@usevenice/integration-import'

type VercelEnv = 'production' | 'preview' | 'development'

/* eslint-disable @typescript-eslint/no-non-null-assertion */
export const commonEnv = zParser(zCommonEnv).parse({
// Need to use fully qualified form of process.env.$VAR for
// webpack DefineEnv that next.js uses to work
// TODO: Maybe we define defaults inside env.ts to avoid duplicating ourselves?
NEXT_PUBLIC_SUPABASE_URL: process.env['NEXT_PUBLIC_SUPABASE_URL']!,
NEXT_PUBLIC_SUPABASE_ANON_KEY: process.env['NEXT_PUBLIC_SUPABASE_ANON_KEY']!,
NEXT_PUBLIC_SENTRY_DSN: process.env['NEXT_PUBLIC_SENTRY_DSN']!,
NEXT_PUBLIC_POSTHOG_WRITEKEY: process.env['NEXT_PUBLIC_POSTHOG_WRITEKEY']!,
NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY:
process.env['NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY']!,
NEXT_PUBLIC_CLERK_SUPABASE_JWT_TEMPLATE_NAME:
process.env['NEXT_PUBLIC_CLERK_SUPABASE_JWT_TEMPLATE_NAME']!,
DEFAULT_CONNECT_ENV: (
{
production: 'production',
preview: 'development',
development: 'sandbox',
} satisfies Record<VercelEnv, EnvName>
)[process.env['NEXT_PUBLIC_VERCEL_ENV'] ?? ''],
})
/* eslint-enable @typescript-eslint/no-non-null-assertion */
export const commonEnv = env

// TODO: Removing providers we are not using so we don't have nearly as much code, at least on the frontend!
// Further perhaps code from supported providers can be loaded dynamically based on
Expand Down
166 changes: 59 additions & 107 deletions apps/app-config/env.ts
Original file line number Diff line number Diff line change
@@ -1,107 +1,59 @@
import {makeId, zEnvName} from '@usevenice/cdk-core'
import {R, z, zEnvVars, zFlattenForEnv} from '@usevenice/util'

import type {PROVIDERS} from './providers'
import {DOCUMENTED_PROVIDERS} from './providers'

// MARK: - Env vars

export const zCommonEnv = zEnvVars({
NEXT_PUBLIC_SUPABASE_URL: z.string(),
NEXT_PUBLIC_SUPABASE_ANON_KEY: z.string(),
NEXT_PUBLIC_SENTRY_DSN: z.string().optional(),
NEXT_PUBLIC_POSTHOG_WRITEKEY: z.string().optional(),
NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: z.string(),
NEXT_PUBLIC_CLERK_SUPABASE_JWT_TEMPLATE_NAME: z.string().default('supabase'),

// Deprecated
// TODO: Deprecate me? prefix with NEXT_PUBLIC please
DEFAULT_CONNECT_ENV: zEnvName.default('sandbox'),

// TODO: Make use of me... prefix with NEXT_PUBLIC please
NODE_ENV: z
.string()
.optional()
.default(process.env['NEXT_PUBLIC_NODE_ENV'] ?? process.env.NODE_ENV),
})

export const zBackendEnv = zEnvVars({
POSTGRES_OR_WEBHOOK_URL: z.string().describe(`
Pass a valid postgres(ql):// url for stateful mode. Will be used Primary database used for metadata and user data storage
Pass a valid http(s):// url for stateless mode. Sync data and metadata be sent to provided URL and you are responsible for your own persistence`),
JWT_SECRET_OR_PUBLIC_KEY: z
.string()
.trim()
.describe('Used for validating authenticity of accessToken'),

CLERK_SECRET_KEY: z.string(),

SENTRY_CRON_MONITOR_ID: z
.string()
.optional()
.describe('Used to monitor the schedule syncs cron job'),
})

/** We would prefer to use `.` but vercel env var name can only be number, letter and underscore... */
const separator = '__'
const getPrefix = (name: string) => makeId('int', name, '')

// Should this be all providers or only dcoumented ones?

export const zFlatConfigByProvider = R.mapToObj(DOCUMENTED_PROVIDERS, (p) => [
p.name,
zFlattenForEnv(p.def.integrationConfig ?? z.unknown(), {
prefix: getPrefix(p.name),
separator,
}),
])

export const zIntegrationEnv = zEnvVars(
R.pipe(
zFlatConfigByProvider,
R.values,
R.map((schema) => schema.innerType().shape),
R.mergeAll,
) as {},
)

export const zAllEnv = zCommonEnv.merge(zBackendEnv).merge(zIntegrationEnv)

// MARK: - Parsing integration configs

/**
* Input env must be raw, so means most likely we are parsing the flatConfig input twice
* for the moment unfortunately... But we need this to support transforms in flatConfig
*/
export function parseIntConfigsFromRawEnv(
env: Record<string, string | undefined> = process.env,
) {
return R.pipe(
R.mapValues(zFlatConfigByProvider, (zFlatConfig, name) => {
const subEnv = R.pipe(
R.pickBy(env, (_v, k) => k.startsWith(getPrefix(name))),
(e) => (R.keys(e).length ? e : undefined), // To get .optional() to work
)
try {
return zFlatConfig.optional().parse(subEnv)
} catch (err) {
if (err instanceof z.ZodError && err.issues[0]) {
const issue = err.issues[0]
// const msg = issue.code === 'invalid_type' && issue.message === 'Required' ? ``
// console.log('subEnv', subEnv, issue)
throw new Error(
`Failed to configure "${name}" provider due to invalid env var "${issue.path.join(
separator,
)}": ${issue.message} [${issue.code}]`,
)
}
}
}),
(configMap) => R.pickBy(configMap, (val) => val !== undefined),
) as {
[k in (typeof PROVIDERS)[number]['name']]?: Extract<
(typeof PROVIDERS)[number],
{name: k}
>['def']['_types']['integrationConfig']
}
}
import {createEnv} from '@t3-oss/env-nextjs'
import {z} from 'zod'

export const envConfig = {
server: {
POSTGRES_OR_WEBHOOK_URL: z.string().describe(`
Pass a valid postgres(ql):// url for stateful mode. Will be used Primary database used for metadata and user data storage
Pass a valid http(s):// url for stateless mode. Sync data and metadata be sent to provided URL and you are responsible for your own persistence`),
JWT_SECRET_OR_PUBLIC_KEY: z
.string()
.trim()
.describe('Used for validating authenticity of accessToken'),

CLERK_SECRET_KEY: z.string(),
SENTRY_CRON_MONITOR_ID: z
.string()
.optional()
.describe('Used to monitor the schedule syncs cron job'),

INNGEST_EVENT_KEY: z.string(),
INNGEST_SIGNING_KEY: z.string(),
},
client: {
NEXT_PUBLIC_SUPABASE_URL: z.string(),
NEXT_PUBLIC_SUPABASE_ANON_KEY: z.string(),
NEXT_PUBLIC_SENTRY_DSN: z.string().optional(),
NEXT_PUBLIC_SENTRY_ORG: z.string().optional(),
NEXT_PUBLIC_POSTHOG_WRITEKEY: z.string().optional(),
NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: z.string(),
NEXT_PUBLIC_CLERK_SUPABASE_JWT_TEMPLATE_NAME: z
.string()
.default('supabase'),
},
runtimeEnv: {
CLERK_SECRET_KEY: process.env['CLERK_SECRET_KEY'],
JWT_SECRET_OR_PUBLIC_KEY: process.env['JWT_SECRET_OR_PUBLIC_KEY'],
POSTGRES_OR_WEBHOOK_URL: process.env['POSTGRES_OR_WEBHOOK_URL'],
SENTRY_CRON_MONITOR_ID: process.env['SENTRY_CRON_MONITOR_ID'],
NEXT_PUBLIC_SENTRY_ORG: process.env['NEXT_PUBLIC_SENTRY_ORG'],
NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY:
process.env['NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY'],
NEXT_PUBLIC_CLERK_SUPABASE_JWT_TEMPLATE_NAME:
process.env['NEXT_PUBLIC_CLERK_SUPABASE_JWT_TEMPLATE_NAME'],
NEXT_PUBLIC_POSTHOG_WRITEKEY: process.env['NEXT_PUBLIC_POSTHOG_WRITEKEY'],
NEXT_PUBLIC_SENTRY_DSN: process.env['NEXT_PUBLIC_SENTRY_DSN'],
NEXT_PUBLIC_SUPABASE_ANON_KEY: process.env['NEXT_PUBLIC_SUPABASE_ANON_KEY'],
NEXT_PUBLIC_SUPABASE_URL: process.env['NEXT_PUBLIC_SUPABASE_URL'],
INNGEST_EVENT_KEY: process.env['INNGEST_EVENT_KEY'],
INNGEST_SIGNING_KEY: process.env['INNGEST_SIGNING_KEY'],
},
onInvalidAccess: (variable: string) => {
throw new Error(
`❌ Attempted to access server-side environment variable ${variable} on the client`,
)
},
skipValidation: !!process.env['SKIP_ENV_VALIDATION'],
} satisfies Parameters<typeof createEnv>[0]

export const env = createEnv(envConfig)
68 changes: 68 additions & 0 deletions apps/app-config/integration-envs.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/** @deprecated. We no longer initialize integration from ENVs, but maybe in clis still? */
import {makeId} from '@usevenice/cdk-core'
import {R, z, zEnvVars, zFlattenForEnv} from '@usevenice/util'

import type {PROVIDERS} from './providers'
import {DOCUMENTED_PROVIDERS} from './providers'

/** We would prefer to use `.` but vercel env var name can only be number, letter and underscore... */
const separator = '__'
const getPrefix = (name: string) => makeId('int', name, '')

// Should this be all providers or only dcoumented ones?

export const zFlatConfigByProvider = R.mapToObj(DOCUMENTED_PROVIDERS, (p) => [
p.name,
zFlattenForEnv(p.def.integrationConfig ?? z.unknown(), {
prefix: getPrefix(p.name),
separator,
}),
])

export const zIntegrationEnv = zEnvVars(
R.pipe(
zFlatConfigByProvider,
R.values,
R.map((schema) => schema.innerType().shape),
R.mergeAll,
) as {},
)

// MARK: - Parsing integration configs

/**
* Input env must be raw, so means most likely we are parsing the flatConfig input twice
* for the moment unfortunately... But we need this to support transforms in flatConfig
*/
export function parseIntConfigsFromRawEnv(
env: Record<string, string | undefined> = process.env,
) {
return R.pipe(
R.mapValues(zFlatConfigByProvider, (zFlatConfig, name) => {
const subEnv = R.pipe(
R.pickBy(env, (_v, k) => k.startsWith(getPrefix(name))),
(e) => (R.keys(e).length ? e : undefined), // To get .optional() to work
)
try {
return zFlatConfig.optional().parse(subEnv)
} catch (err) {
if (err instanceof z.ZodError && err.issues[0]) {
const issue = err.issues[0]
// const msg = issue.code === 'invalid_type' && issue.message === 'Required' ? ``
// console.log('subEnv', subEnv, issue)
throw new Error(
`Failed to configure "${name}" provider due to invalid env var "${issue.path.join(
separator,
)}": ${issue.message} [${issue.code}]`,
)
}
}
}),
(configMap) => R.pickBy(configMap, (val) => val !== undefined),
) as {
[k in (typeof PROVIDERS)[number]['name']]?: Extract<
(typeof PROVIDERS)[number],
{name: k}
>['def']['_types']['integrationConfig']
}
}
3 changes: 2 additions & 1 deletion apps/app-config/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
"dev": "pnpm tsx watch ./_generateDocs.bin.ts"
},
"dependencies": {
"@t3-oss/env-nextjs": "0.3.1",
"@usevenice/cdk-core": "workspace:*",
"@usevenice/cdk-ledger": "workspace:*",
"@usevenice/core-integration-airtable": "workspace:*",
Expand All @@ -22,6 +23,7 @@
"@usevenice/engine-frontend": "workspace:*",
"@usevenice/integration-alphavantage": "workspace:*",
"@usevenice/integration-beancount": "workspace:*",
"@usevenice/integration-brex": "workspace:*",
"@usevenice/integration-expensify": "workspace:*",
"@usevenice/integration-foreceipt": "workspace:*",
"@usevenice/integration-heron": "workspace:*",
Expand All @@ -41,7 +43,6 @@
"@usevenice/integration-toggl": "workspace:*",
"@usevenice/integration-venmo": "workspace:*",
"@usevenice/integration-wise": "workspace:*",
"@usevenice/integration-brex": "workspace:*",
"@usevenice/integration-yodlee": "workspace:*",
"@usevenice/util": "workspace:*",
"chokidar": "3.5.3",
Expand Down
Loading

0 comments on commit 0051111

Please sign in to comment.