From 21841749fc8546e57fac2ca9af15adf61bec0986 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Mon, 27 Feb 2023 18:17:11 +0800 Subject: [PATCH] Move from raw SQL access to GraphQL / REST api for now (#40) * [GraphiQL] Stub in new UI (left old one for now) * [SQL] Stub in "contact us" page for now * [SQL] Updated button icon to be email * [GraphiQL] Update styling & themes * [GraphiQL] Remove title area * [APIs] Add back "Query API" * [Integrations] Add settings button * [Integrations] Plaid settings page * [Integrations] Disable plaid settings for now * [Nav] Ensure nested routes highlight side nav * localStorage.DISABLE_ZOD_FORM_FOR_NOW = true to bypass the zod form * Update view definition to work better with graphql * Hook up graphql editor with the right headers * Experimental rest API explorer via stoplight elements * Moving to the new api-access page * Persist active tab into the query param * Ensure default ledger no longer depends on createDatabaseUser * Renaming old and new api access pages * Add realtime api also * Remove redundant prefix * [API] Tweaks * [Rest] Comment out import since it pollutes global CSS Even without the page loaded into another page * [API] Comment old page for now * [API] Remove Rest explorer for now * [Export] Remove & style sandbox labels * [SQL] Remove old access page for now * [Plaid] Disable preconnect input for now * [Connections] Tweak UI to be easier to read * [API] Show API keys & routes in first tab * [API] Move REST api docs to their own page * [API docs] Punch out to view REST api docs full page * Implement proxy for graphql and rest requests against supabase * [REST] Make docs use dark mode * [REST] Set info section to override Postgres API name * Handle apiKey as well during request proxy * Fix joinPath and add more tests * Use personal access token in the api explorer * Remove token param when proxying * Hooking up the api access page with the correct token * Do not pass the header onwards * Create personal access token if needed * Removing sql endpoint and no more database end user also * Add migration to remove existing database users * Fix migration on vanilla postgres * Declare auth_user as record to make vanilla migration pass * Actually fix migration test * Hide the migrations table from api * Backing up a couple functions we no longer use * Fix ci again, we should use the supabase postgres image --------- Co-authored-by: Rob Phillips --- .github/workflows/validate-workflow.yml | 6 +- apps/app-config/backendConfig.ts | 2 +- apps/app-config/commonConfig.ts | 2 +- .../{server-url.ts => constants.ts} | 8 + .../2023-02-27_1710 json and plv8 test.sql | 32 + apps/cli/admin-queries/createAndDropUser.sql | 15 + .../drop graphile worker and pg_cron.sql | 2 + apps/cli/admin-queries/dropUser.sql | 5 - .../admin-queries/lockdown public role.sql | 29 + .../PageLayout/AuthLayout/NavLink.tsx | 3 +- .../PageLayout/AuthLayout/Sidebar.tsx | 6 +- apps/web/components/ResourceCard.tsx | 2 +- .../api-access/DatabaseAccessCard.tsx | 28 - .../components/api-access/GraphQLExplorer.tsx | 44 + .../QueryOutput/ActionsBar.tsx | 83 - .../QueryOutput/ApiEndpointCard.tsx | 35 - .../QueryOutput/OutputFormat.ts | 4 - .../QueryOutput/OutputFormatTabs.tsx | 22 - .../QueryOutput/OutputTabsKey.ts | 9 - .../QueryOutput/QueryOutput.tsx | 128 - .../QueryOutput/index.ts | 2 - .../QueryOutput/useDatabaseQuery.ts | 144 - .../VeniceDatabaseExplorer.tsx | 98 - .../VeniceDatabaseExplorer/index.ts | 1 - .../VeniceDatabaseExplorer/useSavedQuery.ts | 43 - apps/web/components/api-access/index.ts | 5 +- .../ConnectionCard/ConnectionCard.tsx | 11 +- apps/web/lib/supabase-proxy.ts | 58 + apps/web/package.json | 12 +- apps/web/pages/_document.page.tsx | 2 +- apps/web/pages/api-access.page.tsx | 240 +- .../pages/api/graphql/[[...graphql]].page.ts | 17 + apps/web/pages/api/rest/[[...rest]].page.ts | 17 + apps/web/pages/api/sql.page.ts | 80 - apps/web/pages/connections.page.tsx | 12 +- apps/web/pages/export.page.tsx | 199 -- apps/web/pages/global.css | 74 + apps/web/pages/integrations.page.tsx | 14 +- apps/web/pages/integrations/plaid.page.tsx | 96 + apps/web/pages/rest-explorer.page.tsx | 91 + apps/web/server/api-helpers.ts | 45 +- apps/web/server/procedures.ts | 160 +- .../2023-02-27_remove_database_users.sql | 100 + .../migrations/2023-02-27_revoke_anon.sql | 16 + .../2023-01-07_create-views.sql | 107 +- .../integration-plaid/PlaidProvider.ts | 12 +- packages/engine-frontend/useVenice.tsx | 7 +- packages/ui/components/Tabs.tsx | 2 +- packages/ui/icons/BroadcastIcon.tsx | 17 + packages/ui/icons/EmailIcon.tsx | 13 + packages/ui/icons/IntegrationsIcon.tsx | 5 +- packages/ui/icons/SettingsIcon.tsx | 17 + packages/ui/icons/index.ts | 3 + packages/util/url-utils.spec.ts | 38 +- packages/util/url-utils.ts | 13 +- pnpm-lock.yaml | 2871 ++++++++++++++++- 56 files changed, 3908 insertions(+), 1199 deletions(-) rename apps/app-config/{server-url.ts => constants.ts} (69%) create mode 100644 apps/cli/admin-queries/2023-02-27_1710 json and plv8 test.sql create mode 100644 apps/cli/admin-queries/createAndDropUser.sql create mode 100644 apps/cli/admin-queries/drop graphile worker and pg_cron.sql delete mode 100644 apps/cli/admin-queries/dropUser.sql create mode 100644 apps/cli/admin-queries/lockdown public role.sql delete mode 100644 apps/web/components/api-access/DatabaseAccessCard.tsx create mode 100644 apps/web/components/api-access/GraphQLExplorer.tsx delete mode 100644 apps/web/components/api-access/VeniceDatabaseExplorer/QueryOutput/ActionsBar.tsx delete mode 100644 apps/web/components/api-access/VeniceDatabaseExplorer/QueryOutput/ApiEndpointCard.tsx delete mode 100644 apps/web/components/api-access/VeniceDatabaseExplorer/QueryOutput/OutputFormat.ts delete mode 100644 apps/web/components/api-access/VeniceDatabaseExplorer/QueryOutput/OutputFormatTabs.tsx delete mode 100644 apps/web/components/api-access/VeniceDatabaseExplorer/QueryOutput/OutputTabsKey.ts delete mode 100644 apps/web/components/api-access/VeniceDatabaseExplorer/QueryOutput/QueryOutput.tsx delete mode 100644 apps/web/components/api-access/VeniceDatabaseExplorer/QueryOutput/index.ts delete mode 100644 apps/web/components/api-access/VeniceDatabaseExplorer/QueryOutput/useDatabaseQuery.ts delete mode 100644 apps/web/components/api-access/VeniceDatabaseExplorer/VeniceDatabaseExplorer.tsx delete mode 100644 apps/web/components/api-access/VeniceDatabaseExplorer/index.ts delete mode 100644 apps/web/components/api-access/VeniceDatabaseExplorer/useSavedQuery.ts create mode 100644 apps/web/lib/supabase-proxy.ts create mode 100644 apps/web/pages/api/graphql/[[...graphql]].page.ts create mode 100644 apps/web/pages/api/rest/[[...rest]].page.ts delete mode 100644 apps/web/pages/api/sql.page.ts delete mode 100644 apps/web/pages/export.page.tsx create mode 100644 apps/web/pages/integrations/plaid.page.tsx create mode 100644 apps/web/pages/rest-explorer.page.tsx create mode 100644 integrations/core-integration-postgres/migrations/2023-02-27_remove_database_users.sql create mode 100644 integrations/core-integration-postgres/migrations/2023-02-27_revoke_anon.sql create mode 100644 packages/ui/icons/BroadcastIcon.tsx create mode 100644 packages/ui/icons/EmailIcon.tsx create mode 100644 packages/ui/icons/SettingsIcon.tsx diff --git a/.github/workflows/validate-workflow.yml b/.github/workflows/validate-workflow.yml index 3c12c08a..fbbfcd9f 100644 --- a/.github/workflows/validate-workflow.yml +++ b/.github/workflows/validate-workflow.yml @@ -21,7 +21,7 @@ jobs: # optional (defaults to `5432`) POSTGRES_PORT: 5432 # optional (defaults to `postgres`) - POSTGRES_USER: test + POSTGRES_USER: postgres ports: # maps tcp port 5432 on service container to the host - 5432:5432 @@ -76,7 +76,9 @@ jobs: run: POSTGRES_OR_WEBHOOK_URL=noop NEXT_PUBLIC_SUPABASE_URL=noop NEXT_PUBLIC_SUPABASE_ANON_KEY=noop node --loader tsx ./bin/venice.ts health - name: Run migration check - run: POSTGRES_OR_WEBHOOK_URL=postgres://test:test@localhost:5432/test pnpm migration up + run: POSTGRES_OR_WEBHOOK_URL=postgres://postgres:test@localhost:5432/test pnpm migration up + # To test this with a locally install postgres, run + # psql postgres -c 'drop database if exists test;' && psql postgres -c 'create database test;' && POSTGRES_OR_WEBHOOK_URL=postgres://localhost:5432/test pnpm migration up - name: Run lint run: pnpm run lint diff --git a/apps/app-config/backendConfig.ts b/apps/app-config/backendConfig.ts index 58ac0108..bb769b96 100644 --- a/apps/app-config/backendConfig.ts +++ b/apps/app-config/backendConfig.ts @@ -15,7 +15,7 @@ import {joinPath, R, Rx, zParser} from '@usevenice/util' import {veniceCommonConfig} from './commonConfig' import type {PROVIDERS} from './env' import {parseIntConfigsFromRawEnv, zAllEnv} from './env' -import {getServerUrl} from './server-url' +import {getServerUrl} from './constants' export {Papa} from '@usevenice/integration-import' export {makePostgresClient} from '@usevenice/integration-postgres' diff --git a/apps/app-config/commonConfig.ts b/apps/app-config/commonConfig.ts index ba04dce8..15b5c29d 100644 --- a/apps/app-config/commonConfig.ts +++ b/apps/app-config/commonConfig.ts @@ -3,7 +3,7 @@ import {VeniceProvider} from '@usevenice/engine-frontend' import {joinPath, zParser} from '@usevenice/util' import {PROVIDERS, zCommonEnv} from './env' -import {getServerUrl} from './server-url' +import {getServerUrl} from './constants' export {Papa} from '@usevenice/integration-import' diff --git a/apps/app-config/server-url.ts b/apps/app-config/constants.ts similarity index 69% rename from apps/app-config/server-url.ts rename to apps/app-config/constants.ts index 9e6e3817..ea428bf8 100644 --- a/apps/app-config/server-url.ts +++ b/apps/app-config/constants.ts @@ -15,3 +15,11 @@ export function getServerUrl(req: GetServerSidePropsContext['req'] | null) { }` ) } + +export const graphqlEndpoint = new URL('/api/graphql', getServerUrl(null)) + +export const restEndpoint = new URL('/api/rest', getServerUrl(null)) + +export const xPatHeaderKey = 'x-token' +export const xPatUrlParamKey = 'token' +export const xPatUserMetadataKey = 'apiKey' diff --git a/apps/cli/admin-queries/2023-02-27_1710 json and plv8 test.sql b/apps/cli/admin-queries/2023-02-27_1710 json and plv8 test.sql new file mode 100644 index 00000000..b114c954 --- /dev/null +++ b/apps/cli/admin-queries/2023-02-27_1710 json and plv8 test.sql @@ -0,0 +1,32 @@ +CREATE OR REPLACE FUNCTION public.xjsonb_object_keys(obj jsonb) + RETURNS text[] + LANGUAGE plv8 + IMMUTABLE STRICT +AS $function$ + return Object.keys(obj); +$function$ + +CREATE OR REPLACE FUNCTION public.jsonb_object_keys_to_text_array(_js jsonb) + RETURNS text[] + LANGUAGE sql + IMMUTABLE PARALLEL SAFE STRICT +AS $function$SELECT ARRAY(SELECT jsonb_object_keys(_js))$function$ + +CREATE OR REPLACE FUNCTION public.plv8_test(keys text[], vals text[]) + RETURNS json + LANGUAGE plv8 + IMMUTABLE STRICT +AS $function$ + var o = {}; + for(var i=0; i diff --git a/apps/web/components/PageLayout/AuthLayout/Sidebar.tsx b/apps/web/components/PageLayout/AuthLayout/Sidebar.tsx index ce3598d8..0cd86c8a 100644 --- a/apps/web/components/PageLayout/AuthLayout/Sidebar.tsx +++ b/apps/web/components/PageLayout/AuthLayout/Sidebar.tsx @@ -1,8 +1,7 @@ import { ArrowLeftRightIcon, - DatabaseIcon, + CodeIcon, DocsIcon, - ExportIcon, IntegrationsIcon, ProfileIcon, SupportIcon, @@ -17,11 +16,10 @@ const mainNavigation = [ href: '/connections', icon: ArrowLeftRightIcon, }, - {name: 'Export Data', href: '/export', icon: ExportIcon}, { name: 'API Access', href: '/api-access', - icon: DatabaseIcon, + icon: CodeIcon, }, { name: 'Integrations', diff --git a/apps/web/components/ResourceCard.tsx b/apps/web/components/ResourceCard.tsx index 719f91d9..d29582e3 100644 --- a/apps/web/components/ResourceCard.tsx +++ b/apps/web/components/ResourceCard.tsx @@ -16,7 +16,7 @@ export function ResourceCard(props: ResourceCardProps) {
diff --git a/apps/web/components/api-access/DatabaseAccessCard.tsx b/apps/web/components/api-access/DatabaseAccessCard.tsx deleted file mode 100644 index 820398e5..00000000 --- a/apps/web/components/api-access/DatabaseAccessCard.tsx +++ /dev/null @@ -1,28 +0,0 @@ -import {Input, InstructionCard} from '@usevenice/ui' -import {DatabaseIcon} from '@usevenice/ui/icons' -import {CopyTextButton} from '../CopyTextButton' - -interface DatabaseAccessCardProps { - databaseUrl: string -} - -export function DatabaseAccessCard(props: DatabaseAccessCardProps) { - const {databaseUrl} = props - return ( - -

- Directly connect to Venice's unified database to: -

-
    -
  • Build apps & internal tools
    • -
  • Create dashboards & charts
    • -
  • Use your favorite SQL editor
    • -
  • Write custom data analysis scripts
    • -
-
- - -
- - ) -} diff --git a/apps/web/components/api-access/GraphQLExplorer.tsx b/apps/web/components/api-access/GraphQLExplorer.tsx new file mode 100644 index 00000000..3afdbd16 --- /dev/null +++ b/apps/web/components/api-access/GraphQLExplorer.tsx @@ -0,0 +1,44 @@ +import {createGraphiQLFetcher} from '@graphiql/toolkit' +import {graphqlEndpoint, xPatHeaderKey} from '@usevenice/app-config/constants' +import {useConstant} from '@usevenice/ui' +import {GraphiQL} from 'graphiql' +import 'graphiql/graphiql.css' +import React from 'react' + +export function GraphQLExplorer({pat}: {pat: string}) { + const fetcher = useConstant(() => + createGraphiQLFetcher({url: graphqlEndpoint.pathname}), + ) + const headersString = React.useMemo( + () => JSON.stringify({[xPatHeaderKey]: pat}, null, 4), + [pat], + ) + + return ( +
+ + +
+ + +
+ ) +} diff --git a/apps/web/components/api-access/VeniceDatabaseExplorer/QueryOutput/ActionsBar.tsx b/apps/web/components/api-access/VeniceDatabaseExplorer/QueryOutput/ActionsBar.tsx deleted file mode 100644 index 9c435f7e..00000000 --- a/apps/web/components/api-access/VeniceDatabaseExplorer/QueryOutput/ActionsBar.tsx +++ /dev/null @@ -1,83 +0,0 @@ -import {Button, CircularProgress} from '@usevenice/ui' -import {DownloadIcon, PlayIcon} from '@usevenice/ui/icons' -import {useEffect} from 'react' -import {OutputFormatTabs} from './OutputFormatTabs' -import {OutputTabsKey} from './OutputTabsKey' -import type {DatabaseQuery} from './useDatabaseQuery' - -interface ActionsBarProps { - databaseQuery: DatabaseQuery -} - -export function ActionsBar(props: ActionsBarProps) { - const {databaseQuery} = props - const isFetching = - databaseQuery.csv.isFetching || databaseQuery.json.isFetching - - return ( -
- - {/* OutputControlsActionGroup */} -
- - -
-
- ) -} - -interface ExecuteQueryActionGroupProps { - executeQuery: () => void - isFetching: boolean -} - -function ExecuteQueryActionGroup(props: ExecuteQueryActionGroupProps) { - const {executeQuery, isFetching} = props - - // binds keyboard shortcuts - useEffect(() => { - function handleKeyboardShortcut(event: KeyboardEvent) { - // TODO handle Windows platform - if (event.metaKey && event.key === 'Enter') { - executeQuery() - } - } - document.addEventListener('keydown', handleKeyboardShortcut) - return () => document.removeEventListener('keydown', handleKeyboardShortcut) - }, [executeQuery]) - - return ( -
- - - or hit ⌘ + Enter - -
- ) -} diff --git a/apps/web/components/api-access/VeniceDatabaseExplorer/QueryOutput/ApiEndpointCard.tsx b/apps/web/components/api-access/VeniceDatabaseExplorer/QueryOutput/ApiEndpointCard.tsx deleted file mode 100644 index 8e53fd47..00000000 --- a/apps/web/components/api-access/VeniceDatabaseExplorer/QueryOutput/ApiEndpointCard.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import {InstructionCard} from '@usevenice/ui' -import {CsvFileIcon, JsonFileIcon} from '@usevenice/ui/icons' -import {CopyTextButton} from '../../../CopyTextButton' -import {OutputFormat} from './OutputFormat' - -interface ApiEndpointCardProps { - format: OutputFormat - queryUrl: string -} - -export function ApiEndpointCard(props: ApiEndpointCardProps) { - const {format, queryUrl} = props - const icon = iconByOutputFormat[format] - return ( - -

- Easily call into Venice as a {format.toUpperCase()} API for this query: -

-