New release - Merge branch 'staging'

.github/workflows/validate-workflow.yml

@@ -36,9 +36,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      # - name: Set up tmate session
-      #   uses: mxschmitt/action-tmate@v2
-
       - name: Install Node.js
         uses: actions/setup-node@v3
         with:
@@ -67,6 +64,12 @@ jobs:
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
+      # - name: Set up tmate session
+      #   uses: mxschmitt/action-tmate@v2
+
+      # - name: Setup upterm session
+      #   uses: lhotari/action-upterm@v1
+
       - name: Run type checks
         run: pnpm run typecheck
 

.gitignore

@@ -46,3 +46,5 @@ yarn-error.log*
 /apps/web/__generated__/tailwind.css
 
 *.secret.json
+
+temp/

.vscode/settings.json

@@ -78,5 +78,8 @@
   "tailwindCSS.experimental.configFile": "apps/web/tailwind.config.ts",
   "typescript.format.insertSpaceAfterOpeningAndBeforeClosingNonemptyBraces": false,
   "typescript.tsdk": "./node_modules/typescript/lib",
-  "typescript.enablePromptUseWorkspaceTsdk": true
+  "typescript.enablePromptUseWorkspaceTsdk": true,
+  "workbench.colorCustomizations": {
+    "titleBar.activeBackground": "#0C4B33"
+  }
 }

apps/app-config/_generateDocs.bin.ts

@@ -7,15 +7,17 @@ import {buildUrl, R} from '@usevenice/util'
 
 import {env, envConfig} from './env'
 import {parseIntConfigsFromRawEnv} from './integration-envs'
-import {DOCUMENTED_PROVIDERS} from './providers'
+import {defIntegrations} from './integrations/integrations.def'
 
 const envList = R.pipe(
   {...envConfig.server, ...envConfig.client},
   R.toPairs,
   R.filter(
     ([key]) =>
       !key.startsWith('int') ||
-      DOCUMENTED_PROVIDERS.some((p) => key.startsWith(`int_${p.name}`)),
+      Object.values(defIntegrations).some((p) =>
+        key.startsWith(`int_${p.name}`),
+      ),
   ),
   R.map(([key, schema]) => {
     const cmtLines = R.pipe(

apps/app-config/_generateIntegrationLists.ts

@@ -98,7 +98,11 @@ writePretty(
   `${mergedlist
     .flatMap((int) => {
       const validImports = Object.fromEntries(
-        Object.entries(int.imports).filter(([, v]) => !!v),
+        Object.entries(int.imports)
+          .filter(([, v]) => !!v)
+          // Temp hack because mergedIntegrations are only ever used server side
+          // This avoids server needing to import client side code unnecessarily
+          .filter(([k]) => k !== 'client'),
       )
       return [
         Object.entries(validImports)

apps/app-config/backendConfig.ts

@@ -7,18 +7,20 @@ import {
   mapStandardEntityLink,
   renameAccountLink,
 } from '@usevenice/cdk-ledger'
-import {makePostgresMetaService} from '@usevenice/core-integration-postgres'
 import type {PipelineInput} from '@usevenice/engine-backend'
 import {getContextFactory} from '@usevenice/engine-backend'
+import {makePostgresMetaService} from '@usevenice/integration-postgres'
 import {joinPath, R, Rx} from '@usevenice/util'
 
 import {getServerUrl} from './constants'
 import {env} from './env'
-import {PROVIDERS} from './providers'
+import {mergedIntegrations} from './integrations/integrations.merged'
 
-export {DatabaseError} from '@usevenice/core-integration-postgres/makePostgresClient'
+export {
+  DatabaseError,
+  makePostgresClient,
+} from '@usevenice/integration-postgres/makePostgresClient'
 export {Papa} from '@usevenice/integration-spreadsheet'
-export {makePostgresClient} from '@usevenice/integration-postgres'
 
 export const backendEnv = env
 
@@ -38,15 +40,17 @@ const usePg = env.POSTGRES_OR_WEBHOOK_URL.startsWith('postgres')
 //   VeniceRouter['_def']['mutations']['syncPipeline']
 // >[0]
 export type VeniceInput = PipelineInput<
-  (typeof PROVIDERS)[number],
-  (typeof PROVIDERS)[number]
+  (typeof mergedIntegrations)[keyof typeof mergedIntegrations],
+  (typeof mergedIntegrations)[keyof typeof mergedIntegrations]
 >
 
 export const contextFactory = getContextFactory({
-  providers: PROVIDERS,
+  providers: Object.values(mergedIntegrations),
   // routerUrl: 'http://localhost:3010/api', // apiUrl?
   apiUrl: joinPath(getServerUrl(null), '/api/trpc'),
+  env,
   jwtSecret: env.JWT_SECRET_OR_PUBLIC_KEY,
+  nangoSecretKey: env.NANGO_SECRET_KEY,
   getRedirectUrl: (_, _ctx) => joinPath(getServerUrl(null), '/'),
   getMetaService: (viewer) =>
     makePostgresMetaService({databaseUrl: env.POSTGRES_OR_WEBHOOK_URL, viewer}),

apps/app-config/bootstrap.ts

@@ -1,15 +1,11 @@
 import type {Id} from '@usevenice/cdk-core'
 import {extractId, makeId} from '@usevenice/cdk-core'
-import type {IntegrationInput} from '@usevenice/engine-backend'
 import {flatRouter} from '@usevenice/engine-backend'
 import {getEnvVar} from '@usevenice/util'
 
 import {contextFactory} from './backendConfig'
-import type {PROVIDERS} from './providers'
 import {parseIntConfigsFromRawEnv} from './integration-envs'
 
-export type _ResourceInput = IntegrationInput<(typeof PROVIDERS)[number]>
-
 // TODO: Is this file needed? We can most likely just
 // embed the functionality into venice cli directly...
 export async function bootstrap() {

apps/app-config/env.ts

@@ -19,6 +19,8 @@ Pass a valid http(s):// url for stateless mode. Sync data and metadata be sent t
 
     INNGEST_EVENT_KEY: z.string(),
     INNGEST_SIGNING_KEY: z.string(),
+    NANGO_SECRET_KEY: z.string(),
+    VERCEL_ENV: z.enum(['production', 'preview', 'development']),
   },
   client: {
     NEXT_PUBLIC_SUPABASE_URL: z.string(),
@@ -30,12 +32,16 @@ Pass a valid http(s):// url for stateless mode. Sync data and metadata be sent t
     NEXT_PUBLIC_CLERK_SUPABASE_JWT_TEMPLATE_NAME: z
       .string()
       .default('supabase'),
+    NEXT_PUBLIC_NANGO_PUBLIC_KEY: z.string(),
     NEXT_PUBLIC_COMMANDBAR_ORG_ID: z.string().optional(),
   },
   runtimeEnv: overrideFromLocalStorage({
+    VERCEL_ENV: process.env['VERCEL_ENV'],
     CLERK_SECRET_KEY: process.env['CLERK_SECRET_KEY'],
     INNGEST_EVENT_KEY: process.env['INNGEST_EVENT_KEY'],
     INNGEST_SIGNING_KEY: process.env['INNGEST_SIGNING_KEY'],
+    NANGO_SECRET_KEY: process.env['NANGO_SECRET_KEY'],
+    NEXT_PUBLIC_NANGO_PUBLIC_KEY: process.env['NEXT_PUBLIC_NANGO_PUBLIC_KEY'],
     JWT_SECRET_OR_PUBLIC_KEY: process.env['JWT_SECRET_OR_PUBLIC_KEY'],
     NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY:
       process.env['NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY'],
@@ -59,6 +65,7 @@ Pass a valid http(s):// url for stateless mode. Sync data and metadata be sent t
 } satisfies Parameters<typeof createEnv>[0]
 
 export const env = createEnv(envConfig)
+export type Env = typeof env
 
 // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any
 ;(globalThis as any).env = env

apps/app-config/integration-envs.ts

@@ -1,23 +1,25 @@
 /** @deprecated. We no longer initialize integration from ENVs, but maybe in clis still? */
+import type {IntegrationSchemas, IntHelpers} from '@usevenice/cdk-core'
 import {makeId} from '@usevenice/cdk-core'
 import {R, z, zEnvVars, zFlattenForEnv} from '@usevenice/util'
 
-import type {PROVIDERS} from './providers'
-import {DOCUMENTED_PROVIDERS} from './providers'
+import {defIntegrations} from './integrations/integrations.def'
 
 /** We would prefer to use `.` but vercel env var name can only be number, letter and underscore... */
 const separator = '__'
 const getPrefix = (name: string) => makeId('int', name, '')
 
 // Should this be all providers or only dcoumented ones?
 
-export const zFlatConfigByProvider = R.mapToObj(DOCUMENTED_PROVIDERS, (p) => [
-  p.name,
-  zFlattenForEnv(p.def.integrationConfig ?? z.unknown(), {
-    prefix: getPrefix(p.name),
-    separator,
-  }),
-])
+export const zFlatConfigByProvider = R.mapValues(defIntegrations, (def, name) =>
+  zFlattenForEnv(
+    (def.def as IntegrationSchemas)?.integrationConfig ?? z.unknown(),
+    {
+      prefix: getPrefix(name),
+      separator,
+    },
+  ),
+)
 
 export const zIntegrationEnv = zEnvVars(
   R.pipe(
@@ -60,9 +62,13 @@ export function parseIntConfigsFromRawEnv(
     }),
     (configMap) => R.pickBy(configMap, (val) => val !== undefined),
   ) as {
-    [k in (typeof PROVIDERS)[number]['name']]?: Extract<
-      (typeof PROVIDERS)[number],
-      {name: k}
-    >['def']['_types']['integrationConfig']
+    [k in keyof typeof defIntegrations]?: GetIntConfig<
+      IntHelpers<(typeof defIntegrations)[k]['def']>['_types']
+    >
   }
 }
+
+/** Feels like bit of a hack... */
+type GetIntConfig<T> = T extends {integrationConfig: unknown}
+  ? T['integrationConfig']
+  : {}

apps/app-config/integrations/integrations.def.ts

@@ -1,22 +1,69 @@
 // generated by _generateIntegrationLists.ts. Do not modify by hand
+import {default as integrationAirtable} from '@usevenice/integration-airtable/def'
+import {default as integrationBeancount} from '@usevenice/integration-beancount/def'
 import {default as integrationBrex} from '@usevenice/integration-brex/def'
+import {default as integrationDebug} from '@usevenice/integration-debug/def'
+import {default as integrationFirebase} from '@usevenice/integration-firebase/def'
+import {default as integrationForeceipt} from '@usevenice/integration-foreceipt/def'
+import {default as integrationFs} from '@usevenice/integration-fs/def'
 import {default as integrationHeron} from '@usevenice/integration-heron/def'
+import {default as integrationLunchmoney} from '@usevenice/integration-lunchmoney/def'
 import {default as integrationMercury} from '@usevenice/integration-mercury/def'
 import {default as integrationMerge} from '@usevenice/integration-merge/def'
+import {default as integrationMongodb} from '@usevenice/integration-mongodb/def'
+import {default as integrationMoota} from '@usevenice/integration-moota/def'
 import {default as integrationOnebrick} from '@usevenice/integration-onebrick/def'
 import {default as integrationPlaid} from '@usevenice/integration-plaid/def'
+import {default as integrationPostgres} from '@usevenice/integration-postgres/def'
+import {default as integrationQbo} from '@usevenice/integration-qbo/def'
+import {default as integrationRamp} from '@usevenice/integration-ramp/def'
+import {default as integrationSaltedge} from '@usevenice/integration-saltedge/def'
+import {default as integrationSplitwise} from '@usevenice/integration-splitwise/def'
+import {default as integrationSpreadsheet} from '@usevenice/integration-spreadsheet/def'
 import {default as integrationStripe} from '@usevenice/integration-stripe/def'
 import {default as integrationTeller} from '@usevenice/integration-teller/def'
+import {default as integrationToggl} from '@usevenice/integration-toggl/def'
+import {default as integrationVenmo} from '@usevenice/integration-venmo/def'
+import {default as integrationWebhook} from '@usevenice/integration-webhook/def'
+import {default as integrationWise} from '@usevenice/integration-wise/def'
 import {default as integrationYodlee} from '@usevenice/integration-yodlee/def'
+import type {NonEmptyArray} from '@usevenice/util';
+import { z} from '@usevenice/util'
 
 export const defIntegrations = {
+  airtable: integrationAirtable,
+  beancount: integrationBeancount,
   brex: integrationBrex,
+  debug: integrationDebug,
+  firebase: integrationFirebase,
+  foreceipt: integrationForeceipt,
+  fs: integrationFs,
   heron: integrationHeron,
+  lunchmoney: integrationLunchmoney,
   mercury: integrationMercury,
   merge: integrationMerge,
+  mongodb: integrationMongodb,
+  moota: integrationMoota,
   onebrick: integrationOnebrick,
   plaid: integrationPlaid,
+  postgres: integrationPostgres,
+  qbo: integrationQbo,
+  ramp: integrationRamp,
+  saltedge: integrationSaltedge,
+  splitwise: integrationSplitwise,
+  spreadsheet: integrationSpreadsheet,
   stripe: integrationStripe,
   teller: integrationTeller,
+  toggl: integrationToggl,
+  venmo: integrationVenmo,
+  webhook: integrationWebhook,
+  wise: integrationWise,
   yodlee: integrationYodlee,
 }
+
+// TODO: make sure to generate this
+export const zProviderName = z.enum(
+  Object.keys(defIntegrations) as NonEmptyArray<keyof typeof defIntegrations>,
+)
+
+export type ProviderName = z.infer<typeof zProviderName>