From 47ea8ce4d918a6c2e82782097e07eb1aba09d078 Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Wed, 27 Nov 2024 06:13:20 -0600 Subject: [PATCH] Store PDF documents in db table (#371) * Add form_documents table and an initializeForm service, responsible for creating a form that's initialized with data from a PDF. initializeForm is not currently wired to the UI. * Improve pattern builder docs * Fix test and typing errors; implement localStorage version of addDocument * Wire initializeForm to UI * PDF attachment download working * Fix imports * Rebuild lock file * Revert "Rebuild lock file" This reverts commit 0b0f6c751c9fffb7d21fc69135a7f31a9143d93d. * Remove pdf bytes from blueprint object * Parse/validate Blueprint JSON * Delete associated documents when a form is deleted. * Document support in-browser * Add tests for parseForm * Inject a FormConfig into the forms db adapter * Remove console.log * Call parseStringForm() in getForm db function * Update BrowserFormRepository for interface changes * Fix typo --- apps/spotlight/src/context.ts | 2 + .../store/actions/get-form-session.ts | 1 - packages/common/src/index.ts | 1 + .../20241031103354_form_documents_table.mjs | 21 +++ packages/database/src/clients/kysely/types.ts | 16 +- .../design/src/FormManager/FormList/store.ts | 39 +++-- packages/design/src/FormManager/store.tsx | 12 +- packages/forms/src/blueprint.ts | 6 +- packages/forms/src/builder/index.ts | 12 +- packages/forms/src/builder/parse-form.test.ts | 153 ++++++++++++++++++ packages/forms/src/builder/parse-form.ts | 62 +++++++ .../forms/src/context/browser/form-repo.ts | 112 +++++++++---- packages/forms/src/context/index.ts | 9 +- packages/forms/src/context/test/index.ts | 2 + .../__tests__/doj-pardon-marijuana.test.ts | 3 +- .../src/documents/__tests__/extract.test.ts | 2 +- .../src/documents/__tests__/fill-pdf.test.ts | 3 +- packages/forms/src/documents/document.ts | 36 ++++- packages/forms/src/documents/pdf/extract.ts | 2 +- packages/forms/src/documents/pdf/index.ts | 20 ++- .../forms/src/documents/pdf/parsing-api.ts | 2 +- packages/forms/src/index.ts | 1 + packages/forms/src/patterns/README.md | 9 +- .../patterns/package-download/submit.test.ts | 44 +++-- .../src/patterns/package-download/submit.ts | 37 ++++- .../src/patterns/page-set/submit.test.ts | 78 ++++++--- .../forms/src/patterns/page-set/submit.ts | 6 +- .../forms/src/repository/add-document.test.ts | 33 ++++ packages/forms/src/repository/add-document.ts | 39 +++++ .../forms/src/repository/add-form.test.ts | 6 +- packages/forms/src/repository/add-form.ts | 9 +- .../forms/src/repository/delete-form.test.ts | 128 ++++++++++++++- packages/forms/src/repository/delete-form.ts | 48 ++++-- .../forms/src/repository/get-document.test.ts | 45 ++++++ packages/forms/src/repository/get-document.ts | 39 +++++ .../src/repository/get-form-list.test.ts | 9 +- .../forms/src/repository/get-form-list.ts | 6 +- .../src/repository/get-form-session.test.ts | 9 +- .../forms/src/repository/get-form-session.ts | 6 +- .../forms/src/repository/get-form.test.ts | 17 +- packages/forms/src/repository/get-form.ts | 39 ++--- packages/forms/src/repository/index.ts | 17 +- .../forms/src/repository/save-form.test.ts | 23 ++- packages/forms/src/repository/save-form.ts | 9 +- packages/forms/src/repository/serialize.ts | 21 --- .../repository/upsert-form-session.test.ts | 23 ++- .../src/repository/upsert-form-session.ts | 6 +- packages/forms/src/services/delete-form.ts | 10 +- .../forms/src/services/get-form-session.ts | 15 +- packages/forms/src/services/get-form.test.ts | 2 +- packages/forms/src/services/get-form.ts | 17 +- packages/forms/src/services/index.ts | 5 +- .../src/services/initialize-form.test.ts | 66 ++++++++ .../forms/src/services/initialize-form.ts | 121 ++++++++++++++ packages/forms/src/services/save-form.test.ts | 7 +- packages/forms/src/services/save-form.ts | 14 +- packages/forms/src/services/submit-form.ts | 33 ++-- packages/forms/src/session.ts | 1 - packages/forms/src/submission.ts | 16 +- packages/forms/src/testing.ts | 17 +- packages/forms/src/types.ts | 2 +- .../src/{documents/util.ts => util/base64.ts} | 15 +- packages/server/src/config/services.ts | 7 +- packages/server/src/lib/api-client.ts | 39 ++++- packages/server/src/lib/attachments.ts | 45 ------ packages/server/src/pages/api/forms/index.ts | 5 +- packages/server/src/pages/forms/[id].astro | 18 ++- 67 files changed, 1357 insertions(+), 321 deletions(-) create mode 100644 packages/database/migrations/20241031103354_form_documents_table.mjs create mode 100644 packages/forms/src/builder/parse-form.test.ts create mode 100644 packages/forms/src/builder/parse-form.ts create mode 100644 packages/forms/src/repository/add-document.test.ts create mode 100644 packages/forms/src/repository/add-document.ts create mode 100644 packages/forms/src/repository/get-document.test.ts create mode 100644 packages/forms/src/repository/get-document.ts delete mode 100644 packages/forms/src/repository/serialize.ts create mode 100644 packages/forms/src/services/initialize-form.test.ts create mode 100644 packages/forms/src/services/initialize-form.ts rename packages/forms/src/{documents/util.ts => util/base64.ts} (65%) delete mode 100644 packages/server/src/lib/attachments.ts diff --git a/apps/spotlight/src/context.ts b/apps/spotlight/src/context.ts index 0059da9c..b55f264d 100644 --- a/apps/spotlight/src/context.ts +++ b/apps/spotlight/src/context.ts @@ -2,6 +2,7 @@ import { type FormConfig, type FormService, createFormService, + parsePdf, } from '@atj/forms'; import { defaultFormConfig } from '@atj/forms'; import { BrowserFormRepository } from '@atj/forms/context'; @@ -43,6 +44,7 @@ const createAppFormService = () => { repository, config: defaultFormConfig, isUserLoggedIn: () => true, + parsePdf, }); } else { return createTestBrowserFormService(); diff --git a/apps/spotlight/src/features/form-page/store/actions/get-form-session.ts b/apps/spotlight/src/features/form-page/store/actions/get-form-session.ts index be02ffbc..a3083cca 100644 --- a/apps/spotlight/src/features/form-page/store/actions/get-form-session.ts +++ b/apps/spotlight/src/features/form-page/store/actions/get-form-session.ts @@ -43,7 +43,6 @@ export const getFormSession: GetFormSession = async (ctx, opts) => { }, }); } else { - console.log('using session', result.data.data); ctx.setState({ formSessionResponse: { status: 'loaded', diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts index 5f4ab061..58865a89 100644 --- a/packages/common/src/index.ts +++ b/packages/common/src/index.ts @@ -7,6 +7,7 @@ export type Result = Success | Failure; export type VoidResult = VoidSuccess | Failure; export const success = (data: T): Success => ({ success: true, data }); +export const voidSuccess: VoidSuccess = { success: true }; export const failure = (error: E): Failure => ({ success: false, error }); export { en as enLocale } from './locales/en/app.js'; diff --git a/packages/database/migrations/20241031103354_form_documents_table.mjs b/packages/database/migrations/20241031103354_form_documents_table.mjs new file mode 100644 index 00000000..85ddc3f2 --- /dev/null +++ b/packages/database/migrations/20241031103354_form_documents_table.mjs @@ -0,0 +1,21 @@ +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +export async function up(knex) { + await knex.schema.createTable('form_documents', table => { + table.uuid('id').primary(); + table.string('type').notNullable(); + table.string('file_name').notNullable(); + table.binary('data').notNullable(); + table.string('extract').notNullable(); + }); +} + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +export async function down(knex) { + await knex.schema.dropTableIfExists('form_documents'); +} diff --git a/packages/database/src/clients/kysely/types.ts b/packages/database/src/clients/kysely/types.ts index 80c04b27..0c54b490 100644 --- a/packages/database/src/clients/kysely/types.ts +++ b/packages/database/src/clients/kysely/types.ts @@ -1,4 +1,5 @@ import type { + ColumnType, Generated, Insertable, Kysely, @@ -14,7 +15,9 @@ export interface Database { sessions: SessionsTable; forms: FormsTable; form_sessions: FormSessionsTable; + form_documents: FormDocumentsTable; } +export type DatabaseClient = Kysely; interface UsersTable { id: string; @@ -48,8 +51,6 @@ export type FormsTableSelectable = Selectable; export type FormsTableInsertable = Insertable; export type FormsTableUpdateable = Updateable; -export type DatabaseClient = Kysely; - interface FormSessionsTable { id: string; form_id: string; @@ -60,3 +61,14 @@ interface FormSessionsTable { export type FormSessionsTableSelectable = Selectable; export type FormSessionsTableInsertable = Insertable; export type FormSessionsTableUpdateable = Updateable; + +interface FormDocumentsTable { + id: string; + type: string; + data: ColumnType; + file_name: string; + extract: string; +} +export type FormDocumentsTableSelectable = Selectable; +export type FormDocumentsTableInsertable = Insertable; +export type FormDocumentsTableUpdateable = Updateable; diff --git a/packages/design/src/FormManager/FormList/store.ts b/packages/design/src/FormManager/FormList/store.ts index bb545437..2bd36d54 100644 --- a/packages/design/src/FormManager/FormList/store.ts +++ b/packages/design/src/FormManager/FormList/store.ts @@ -1,6 +1,6 @@ import { type StateCreator } from 'zustand'; -import { BlueprintBuilder } from '@atj/forms'; +import { BlueprintBuilder, uint8ArrayToBase64 } from '@atj/forms'; import { type FormManagerContext } from '../../FormManager/index.js'; import { type Result, failure } from '@atj/common'; @@ -23,18 +23,17 @@ export const createFormListSlice = () => ({ context, createNewFormByPDFUrl: async url => { - const data = await fetchUint8Array(`${context.baseUrl}${url}`); - - const builder = new BlueprintBuilder(context.config); - builder.setFormSummary({ - title: url, - description: '', - }); - await builder.addDocument({ - name: url, - data, + const data = await fetchAsBase64(`${context.baseUrl}${url}`); + const result = await context.formService.initializeForm({ + summary: { + title: url, + description: '', + }, + document: { + fileName: url, + data, + }, }); - const result = await context.formService.addForm(builder.form); if (result.success) { return { success: true, @@ -51,7 +50,16 @@ export const createFormListSlice = description: '', }); await builder.addDocument(fileDetails); - const result = await context.formService.addForm(builder.form); + const result = await context.formService.initializeForm({ + summary: { + title: fileDetails.name, + description: '', + }, + document: { + fileName: fileDetails.name, + data: await uint8ArrayToBase64(fileDetails.data), + }, + }); if (result.success) { return { success: true, @@ -63,8 +71,9 @@ export const createFormListSlice = }, }); -const fetchUint8Array = async (url: string) => { +const fetchAsBase64 = async (url: string) => { const response = await fetch(url); const blob = await response.blob(); - return new Uint8Array(await blob.arrayBuffer()); + const data = new Uint8Array(await blob.arrayBuffer()); + return uint8ArrayToBase64(data); }; diff --git a/packages/design/src/FormManager/store.tsx b/packages/design/src/FormManager/store.tsx index 6479ad4f..625893e0 100644 --- a/packages/design/src/FormManager/store.tsx +++ b/packages/design/src/FormManager/store.tsx @@ -8,7 +8,7 @@ import { import { createContext } from 'zustand-utils'; import { type Result, failure } from '@atj/common'; -import { type FormSession, type Blueprint, BlueprintBuilder } from '@atj/forms'; +import { type FormSession, type Blueprint } from '@atj/forms'; import { type FormListSlice, createFormListSlice } from './FormList/store.js'; import { type FormEditSlice, createFormEditSlice } from './FormEdit/store.js'; @@ -79,12 +79,12 @@ const createFormManagerSlice = inProgress: false, }, createNewForm: async function () { - const builder = new BlueprintBuilder(context.config); - builder.setFormSummary({ - title: `My form - ${new Date().toISOString()}`, - description: '', + const result = await context.formService.initializeForm({ + summary: { + title: `My form - ${new Date().toISOString()}`, + description: '', + }, }); - const result = await context.formService.addForm(builder.form); if (!result.success) { return failure(result.error.message); } diff --git a/packages/forms/src/blueprint.ts b/packages/forms/src/blueprint.ts index 555a6862..4bcfae75 100644 --- a/packages/forms/src/blueprint.ts +++ b/packages/forms/src/blueprint.ts @@ -71,11 +71,11 @@ export const createForm = ( patterns: [ { id: 'root', - type: 'sequence', + type: 'page-set', data: { - patterns: [], + pages: [], }, - } satisfies SequencePattern, + } satisfies PageSetPattern, ], root: 'root', } diff --git a/packages/forms/src/builder/index.ts b/packages/forms/src/builder/index.ts index 5f109702..74323ccb 100644 --- a/packages/forms/src/builder/index.ts +++ b/packages/forms/src/builder/index.ts @@ -9,7 +9,7 @@ import { removePatternFromBlueprint, updateFormSummary, } from '../blueprint.js'; -import { addDocument } from '../documents/document.js'; +import { addDocument, addParsedPdfToForm } from '../documents/document.js'; import type { FormErrors } from '../error.js'; import { createDefaultPattern, @@ -23,6 +23,7 @@ import { import { type FieldsetPattern } from '../patterns/fieldset/config.js'; import { type PageSetPattern } from '../patterns/page-set/config.js'; import type { Blueprint, FormSummary } from '../types.js'; +import type { ParsedPdf } from '../documents/pdf/parsing-api.js'; export class BlueprintBuilder { bp: Blueprint; @@ -47,6 +48,15 @@ export class BlueprintBuilder { this.bp = updatedForm; } + async addDocumentRef(opts: { id: string; extract: ParsedPdf }) { + const { updatedForm } = await addParsedPdfToForm(this.form, { + id: opts.id, + label: opts.extract.title, + extract: opts.extract, + }); + this.bp = updatedForm; + } + addPage() { const newPage = createDefaultPattern(this.config, 'page'); this.bp = addPageToPageSet(this.form, newPage); diff --git a/packages/forms/src/builder/parse-form.test.ts b/packages/forms/src/builder/parse-form.test.ts new file mode 100644 index 00000000..b70aafd4 --- /dev/null +++ b/packages/forms/src/builder/parse-form.test.ts @@ -0,0 +1,153 @@ +import { describe, it, expect } from 'vitest'; +import { failure, success } from '@atj/common'; + +import { parseForm, parseFormString } from './parse-form'; +import { defaultFormConfig, type InputPattern } from '../patterns'; +import type { Blueprint } from '../types'; + +describe('parseForm', () => { + it('should return success when form data is valid', () => { + const formData: Blueprint = { + summary: { + title: 'Test Title', + description: 'Test Description', + }, + root: 'rootValue', + patterns: { + validPattern: { + type: 'input', + id: 'validPattern', + data: { + label: 'label', + required: true, + maxLength: 100, + }, + } satisfies InputPattern, + }, + outputs: [ + { + id: 'output1', + path: 'path/to/output', + fields: { + field1: { + type: 'TextField', + name: 'name', + label: 'label', + value: 'value', + required: true, + }, + }, + formFields: { + formField1: 'formValue1', + }, + }, + ], + }; + + const result = parseForm(defaultFormConfig, formData); + expect(result).toEqual(success(formData)); + }); + + it('should return failure when form data is invalid', () => { + const formData = { + summary: { + title: 'Test Title', + description: 'Test Description', + }, + root: 'rootValue', + patterns: { + invalidPattern: { + type: 'invalidPattern', + data: {}, + }, + }, + outputs: [ + { + id: 'output1', + path: 'path/to/output', + fields: { + field1: 'value1', + }, + formFields: { + formField1: 'formValue1', + }, + }, + ], + }; + + const result = parseForm(defaultFormConfig, formData); + expect(result.success).toEqual(false); + }); +}); + +describe('parseFormString', () => { + it('should return success when JSON string is valid', () => { + const jsonString = JSON.stringify({ + summary: { + title: 'Test Title', + description: 'Test Description', + }, + root: 'rootValue', + patterns: { + validPattern: { + type: 'input', + id: 'validPattern', + data: { + label: 'label', + required: true, + maxLength: 100, + initial: '', + }, + } satisfies InputPattern, + }, + outputs: [], + } satisfies Blueprint); + + const result = parseFormString(defaultFormConfig, jsonString); + expect(result).toEqual(success(JSON.parse(jsonString))); + }); + + it('should return failure when JSON string is invalid', () => { + const jsonString = JSON.stringify({ + summary: { + title: 'Test Title', + description: 'Test Description', + }, + root: 'rootValue', + patterns: { + invalidPattern: { + type: 'invalidPattern', + data: {}, + }, + }, + outputs: [ + { + id: 'output1', + path: 'path/to/output', + fields: { + field1: 'value1', + }, + formFields: { + formField1: 'formValue1', + }, + }, + ], + }); + + const result = parseFormString(defaultFormConfig, jsonString); + expect(result).toEqual({ + success: false, + error: + '[\n' + + ' {\n' + + ' "code": "custom",\n' + + ' "message": "Invalid pattern",\n' + + ' "path": [\n' + + ' "patterns",\n' + + ' "invalidPattern"\n' + + ' ]\n' + + ' }\n' + + ']', + }); + }); +}); diff --git a/packages/forms/src/builder/parse-form.ts b/packages/forms/src/builder/parse-form.ts new file mode 100644 index 00000000..ea86f1f2 --- /dev/null +++ b/packages/forms/src/builder/parse-form.ts @@ -0,0 +1,62 @@ +import * as z from 'zod'; + +import { failure, success, type Result } from '@atj/common'; +import type { FormConfig } from '../pattern'; +import type { Blueprint } from '../types'; + +export const parseForm = (config: FormConfig, obj: any): Result => { + const formSchema = createFormSchema(config); + const result = formSchema.safeParse(obj); + if (result.error) { + return failure(result.error.message); + } + return success(result.data); +}; + +export const parseFormString = ( + config: FormConfig, + json: string +): Result => { + return parseForm(config, JSON.parse(json)); +}; + +const createFormSchema = (config: FormConfig) => { + return z.object({ + summary: z.object({ + title: z.string(), + description: z.string(), + }), + root: z.string(), + patterns: z.record( + z.string(), + z.any().refine( + val => { + const patternConfig = config.patterns[val?.type]; + if (!patternConfig) { + return false; + } + const result = patternConfig.parseConfigData(val?.data); + if (!result.success) { + const message = Object.values(result.error) + .map(err => err.message || '') + .join(', '); + console.error(val?.type, result.error); + console.error(`Pattern config error: ${message}`); + } + return result.success; + }, + { + message: 'Invalid pattern', + } + ) + ), + outputs: z.array( + z.object({ + id: z.string(), + path: z.string(), + fields: z.record(z.string(), z.any()), + formFields: z.record(z.string(), z.string()), + }) + ), + }); +}; diff --git a/packages/forms/src/context/browser/form-repo.ts b/packages/forms/src/context/browser/form-repo.ts index c39566ec..9e4107db 100644 --- a/packages/forms/src/context/browser/form-repo.ts +++ b/packages/forms/src/context/browser/form-repo.ts @@ -1,8 +1,15 @@ -import { type Result, type VoidResult, failure } from '@atj/common'; - -import { FormSession, FormSessionId, type Blueprint } from '../../index.js'; +import { type Result, type VoidResult, failure, success } from '@atj/common'; + +import { + FormSession, + FormSessionId, + type Blueprint, + type DocumentFieldMap, +} from '../../index.js'; import { FormRepository } from '../../repository/index.js'; +import type { ParsedPdf } from '../../documents/pdf/parsing-api.js'; +const documentKey = (id: string) => `documents/${id}`; const formKey = (formId: string) => `forms/${formId}`; const isFormKey = (key: string) => key.startsWith('forms/'); const getFormIdFromKey = (key: string) => { @@ -73,20 +80,22 @@ export class BrowserFormRepository implements FormRepository { }; } - async deleteForm(formId: string): Promise { + async deleteForm( + formId: string + ): Promise> { this.storage.removeItem(formKey(formId)); return { success: true }; } - async getForm(id?: string): Promise { + async getForm(id?: string): Promise> { if (!this.storage || !id) { - return null; + return success(null); } const formString = this.storage.getItem(`forms/${id}`); if (!formString) { - return null; + return success(null); } - return parseStringForm(formString); + return Promise.resolve(success(JSON.parse(formString))); } async getFormList(): Promise< @@ -98,14 +107,17 @@ export class BrowserFormRepository implements FormRepository { } return Promise.all( forms.map(async key => { - const form = await this.getForm(key); - if (form === null) { + const formResult = await this.getForm(key); + if (!formResult.success) { + throw new Error('Error getting form'); + } + if (formResult.data === null) { throw new Error('key mismatch'); } return { id: key, - title: form.summary.title, - description: form.summary.description, + title: formResult.data.summary.title, + description: formResult.data.summary.description, }; }) ); @@ -113,12 +125,55 @@ export class BrowserFormRepository implements FormRepository { async saveForm(formId: string, form: Blueprint): Promise { try { - this.storage.setItem(formKey(formId), stringifyForm(form)); + this.storage.setItem(formKey(formId), JSON.stringify(form)); } catch { return failure(`error saving '${formId}' to storage`); } return { success: true }; } + + addDocument(document: { + fileName: string; + data: Uint8Array; + extract: { parsedPdf: ParsedPdf; fields: DocumentFieldMap }; + }) { + const documentId = crypto.randomUUID(); + const data = uint8ArrayToBase64(document.data); + this.storage.setItem( + documentKey(documentId), + JSON.stringify({ + id: documentId, + type: 'pdf', + file_name: document.fileName, + data, + extract: JSON.stringify(document.extract), + }) + ); + return Promise.resolve( + success({ + id: documentId, + }) + ); + } + + getDocument(id: string): Promise< + Result<{ + id: string; + data: Uint8Array; + path: string; + fields: DocumentFieldMap; + }> + > { + const value = this.storage.getItem(documentKey(id)); + if (value === null) { + return Promise.resolve(failure(`Document with id ${id} not found`)); + } + const json = JSON.parse(value); + return Promise.resolve({ + ...json, + data: base64ToUint8Array(json.data), + }); + } } export const getFormList = (storage: Storage) => { @@ -138,7 +193,7 @@ export const getFormList = (storage: Storage) => { export const saveForm = (storage: Storage, formId: string, form: Blueprint) => { try { - storage.setItem(formKey(formId), stringifyForm(form)); + storage.setItem(formKey(formId), JSON.stringify(form)); } catch { return { success: false as const, @@ -150,17 +205,6 @@ export const saveForm = (storage: Storage, formId: string, form: Blueprint) => { }; }; -const stringifyForm = (form: Blueprint) => { - return JSON.stringify({ - ...form, - outputs: form.outputs.map(output => ({ - ...output, - // TODO: we probably want to do this somewhere in the documents module - data: uint8ArrayToBase64(output.data), - })), - }); -}; - const parseStringForm = (formString: string): Blueprint => { const form = JSON.parse(formString) as Blueprint; return { @@ -181,12 +225,20 @@ const uint8ArrayToBase64 = (buffer: Uint8Array): string => { return btoa(binary); }; +const fixBase64 = (base64: string): string => { + const padding = base64.length % 4; + if (padding === 2) return base64 + '=='; + if (padding === 3) return base64 + '='; + return base64; +}; + const base64ToUint8Array = (base64: string): Uint8Array => { - const binaryString = atob(base64); - const len = binaryString.length; - const bytes = new Uint8Array(len); + const fixedBase64 = fixBase64(base64); + const binary = atob(fixedBase64); + const len = binary.length; + const buffer = new Uint8Array(len); for (let i = 0; i < len; i++) { - bytes[i] = binaryString.charCodeAt(i); + buffer[i] = binary.charCodeAt(i); } - return bytes; + return buffer; }; diff --git a/packages/forms/src/context/index.ts b/packages/forms/src/context/index.ts index 910f3f6e..d73929c7 100644 --- a/packages/forms/src/context/index.ts +++ b/packages/forms/src/context/index.ts @@ -1,10 +1,13 @@ -import { type FormConfig } from '../pattern.js'; -import { type FormRepository } from '../repository/index.js'; +import type { ParsePdf } from '../documents/index.js'; +import type { FormConfig } from '../pattern.js'; +import type { FormRepository } from '../repository/index.js'; -export { createTestBrowserFormService } from './test/index.js'; export { BrowserFormRepository } from './browser/form-repo.js'; +export { createTestBrowserFormService } from './test/index.js'; + export type FormServiceContext = { repository: FormRepository; config: FormConfig; isUserLoggedIn: () => boolean; + parsePdf: ParsePdf; }; diff --git a/packages/forms/src/context/test/index.ts b/packages/forms/src/context/test/index.ts index 6e4cc835..8122c05d 100644 --- a/packages/forms/src/context/test/index.ts +++ b/packages/forms/src/context/test/index.ts @@ -1,4 +1,5 @@ import { BrowserFormRepository } from '../browser/form-repo.js'; +import { parsePdf } from '../../documents/pdf/index.js'; import { defaultFormConfig } from '../../patterns/index.js'; import { type FormService, createFormService } from '../../services/index.js'; @@ -14,6 +15,7 @@ export const createTestBrowserFormService = ( repository, config: defaultFormConfig, isUserLoggedIn: () => true, + parsePdf, }); if (testData) { Object.entries(testData).forEach(([id, blueprint]) => { diff --git a/packages/forms/src/documents/__tests__/doj-pardon-marijuana.test.ts b/packages/forms/src/documents/__tests__/doj-pardon-marijuana.test.ts index 1b828d4a..48ce61aa 100644 --- a/packages/forms/src/documents/__tests__/doj-pardon-marijuana.test.ts +++ b/packages/forms/src/documents/__tests__/doj-pardon-marijuana.test.ts @@ -3,7 +3,8 @@ import { describe, expect, test } from 'vitest'; import { Success } from '@atj/common'; import { type DocumentFieldMap } from '../index.js'; -import { fillPDF, getDocumentFieldData } from '../pdf/index.js'; +import { fillPDF } from '../pdf/index.js'; +import { getDocumentFieldData } from '../pdf/extract.js'; import { loadSamplePDF } from './sample-data.js'; diff --git a/packages/forms/src/documents/__tests__/extract.test.ts b/packages/forms/src/documents/__tests__/extract.test.ts index ccdaef1f..a2ec418b 100644 --- a/packages/forms/src/documents/__tests__/extract.test.ts +++ b/packages/forms/src/documents/__tests__/extract.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; -import { getDocumentFieldData } from '../index.js'; import { loadSamplePDF } from './sample-data.js'; +import { getDocumentFieldData } from '../pdf/extract.js'; describe('PDF form field extraction', () => { it('extracts data from California UD-105 form', async () => { diff --git a/packages/forms/src/documents/__tests__/fill-pdf.test.ts b/packages/forms/src/documents/__tests__/fill-pdf.test.ts index 8fbc5068..4ec57684 100644 --- a/packages/forms/src/documents/__tests__/fill-pdf.test.ts +++ b/packages/forms/src/documents/__tests__/fill-pdf.test.ts @@ -2,8 +2,9 @@ import { beforeAll, describe, expect, it } from 'vitest'; import { type Failure, type Success } from '@atj/common'; -import { getDocumentFieldData, fillPDF } from '../index.js'; +import { fillPDF } from '../index.js'; import { loadSamplePDF } from './sample-data.js'; +import { getDocumentFieldData } from '../pdf/extract.js'; describe('PDF form filler', () => { let pdfBytes: Uint8Array; diff --git a/packages/forms/src/documents/document.ts b/packages/forms/src/documents/document.ts index 48b5fe08..2338834b 100644 --- a/packages/forms/src/documents/document.ts +++ b/packages/forms/src/documents/document.ts @@ -10,18 +10,46 @@ import { type AttachmentPattern } from '../patterns/attachment/config.js'; import { attachmentFileTypeMimes } from '../patterns/attachment/file-type-options.js'; import { type SequencePattern } from '../patterns/sequence.js'; import { type Blueprint } from '../types.js'; +import { getDocumentFieldData } from './pdf/extract.js'; -import { type PDFDocument, getDocumentFieldData } from './pdf/index.js'; +import { type PDFDocument } from './pdf/index.js'; import { type FetchPdfApiResponse, - processApiResponse, + type ParsedPdf, fetchPdfApiResponse, + processApiResponse, } from './pdf/parsing-api.js'; import { type DocumentFieldMap } from './types.js'; export type DocumentTemplate = PDFDocument; +export const addParsedPdfToForm = async ( + form: Blueprint, + document: { + id: string; + label: string; + extract: ParsedPdf; + } +) => { + form = addPatternMap(form, document.extract.patterns, document.extract.root); + const updatedForm = addFormOutput(form, { + id: document.id, + path: document.label, + fields: document.extract.outputs, + formFields: Object.fromEntries( + Object.keys(document.extract.outputs).map(output => { + return [output, document.extract.outputs[output].name]; + }) + ), + }); + return { + newFields: document.extract.outputs, + updatedForm, + errors: document.extract.errors, + }; +}; + export const addDocument = async ( form: Blueprint, fileDetails: { @@ -43,7 +71,7 @@ export const addDocument = async ( }); form = addPatternMap(form, parsedPdf.patterns, parsedPdf.root); const updatedForm = addFormOutput(form, { - data: fileDetails.data, + id: 'document-1', // TODO: generate a unique ID path: fileDetails.name, fields: parsedPdf.outputs, formFields: Object.fromEntries( @@ -60,7 +88,7 @@ export const addDocument = async ( } else { const formWithFields = addDocumentFieldsToForm(form, fields); const updatedForm = addFormOutput(formWithFields, { - data: fileDetails.data, + id: 'document-1', // TODO: generate a unique ID path: fileDetails.name, fields, // TODO: for now, reuse the field IDs from the PDF. we need to generate diff --git a/packages/forms/src/documents/pdf/extract.ts b/packages/forms/src/documents/pdf/extract.ts index 6b619fe7..099dfb3c 100644 --- a/packages/forms/src/documents/pdf/extract.ts +++ b/packages/forms/src/documents/pdf/extract.ts @@ -10,7 +10,7 @@ import { PDFRadioGroup, } from 'pdf-lib'; -import { stringToBase64 } from '../util.js'; +import { stringToBase64 } from '../../util/base64.js'; import type { DocumentFieldValue, DocumentFieldMap } from '../types.js'; // TODO: copied from pdf-lib acrofield internals, check if it's already exposed outside of acroform somewhere diff --git a/packages/forms/src/documents/pdf/index.ts b/packages/forms/src/documents/pdf/index.ts index 1cdb6f77..edafb097 100644 --- a/packages/forms/src/documents/pdf/index.ts +++ b/packages/forms/src/documents/pdf/index.ts @@ -1,4 +1,11 @@ -export { getDocumentFieldData } from './extract.js'; +import { getDocumentFieldData } from './extract.js'; +import { + type ParsedPdf, + fetchPdfApiResponse, + processApiResponse, +} from './parsing-api.js'; +import type { DocumentFieldMap } from '../types.js'; + export * from './generate.js'; export { generateDummyPDF } from './generate-dummy.js'; @@ -21,3 +28,14 @@ export type PDFFieldType = | 'RadioGroup' | 'Paragraph' | 'RichText'; + +export type ParsePdf = ( + pdf: Uint8Array +) => Promise<{ parsedPdf: ParsedPdf; fields: DocumentFieldMap }>; + +export const parsePdf: ParsePdf = async (pdfBytes: Uint8Array) => { + const fields = await getDocumentFieldData(pdfBytes); + const apiResponse = await fetchPdfApiResponse(pdfBytes); + const parsedPdf = await processApiResponse(apiResponse); + return { parsedPdf, fields }; +}; diff --git a/packages/forms/src/documents/pdf/parsing-api.ts b/packages/forms/src/documents/pdf/parsing-api.ts index 58b3e616..d24b9a86 100644 --- a/packages/forms/src/documents/pdf/parsing-api.ts +++ b/packages/forms/src/documents/pdf/parsing-api.ts @@ -9,7 +9,7 @@ import { type CheckboxPattern } from '../../patterns/checkbox.js'; import { type RadioGroupPattern } from '../../patterns/radio-group.js'; import { RichTextPattern } from '../../patterns/rich-text.js'; -import { uint8ArrayToBase64 } from '../util.js'; +import { uint8ArrayToBase64 } from '../../util/base64.js'; import { type DocumentFieldMap } from '../types.js'; import { createPattern, diff --git a/packages/forms/src/index.ts b/packages/forms/src/index.ts index 4e28de04..11200c7c 100644 --- a/packages/forms/src/index.ts +++ b/packages/forms/src/index.ts @@ -8,6 +8,7 @@ export * from './patterns/index.js'; export * from './response.js'; export * from './session.js'; export * from './types.js'; +export * from './util/base64.js'; export { type FormService, createFormService } from './services/index.js'; export { defaultFormConfig, diff --git a/packages/forms/src/patterns/README.md b/packages/forms/src/patterns/README.md index 56daa04b..25b50e76 100644 --- a/packages/forms/src/patterns/README.md +++ b/packages/forms/src/patterns/README.md @@ -39,11 +39,14 @@ const input: InputPattern = { } ``` +... or patterns may be created with the builder's object-oriented interface: + ```typescript -const input = new InputPatternBuilder(); +const input1 = new InputPatternBuilder(); +const input2 = new InputPatternBuilder(); const page1 = new Page({ title: 'Page 1', patterns: [input1.id] }); -const pageSet = new PageSet({ pages: [page1.id] }, 'page-set'); -const page2 = new Page({ title: 'Page 2', patterns: [input1.id] }); +const pageSet = new PageSet({ pages: [page1.id] }); +const page2 = new Page({ title: 'Page 2', patterns: [input2.id] }); pageSet.addPage(page2) // Construct the pattern objects diff --git a/packages/forms/src/patterns/package-download/submit.test.ts b/packages/forms/src/patterns/package-download/submit.test.ts index 53d57a9e..c0815231 100644 --- a/packages/forms/src/patterns/package-download/submit.test.ts +++ b/packages/forms/src/patterns/package-download/submit.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { failure } from '@atj/common'; +import { failure, success } from '@atj/common'; import { type Blueprint, type FormSession, defaultFormConfig } from '../..'; @@ -18,11 +18,17 @@ describe('downloadPackageHandler', async () => { data: { errors: {}, values: {} }, route: { url: '#', params: {} }, }; - const result = await downloadPackageHandler(defaultFormConfig, { - pattern: new PackageDownload({ text: 'Download now!' }).toPattern(), - session, - data: {}, - }); + const result = await downloadPackageHandler( + { + config: defaultFormConfig, + getDocument: () => Promise.resolve(failure('Document not found')), + }, + { + pattern: new PackageDownload({ text: 'Download now!' }).toPattern(), + session, + data: {}, + } + ); expect(result).toEqual(failure('Form is not complete')); }); @@ -38,11 +44,25 @@ describe('downloadPackageHandler', async () => { }, route: { url: '#', params: {} }, }; - const result = await downloadPackageHandler(defaultFormConfig, { - pattern: new PackageDownload({ text: 'Download now!' }).toPattern(), - session, - data: {}, - }); + const result = await downloadPackageHandler( + { + config: defaultFormConfig, + getDocument: async () => + success({ + id: 'id', + data: await loadSamplePDF( + 'doj-pardon-marijuana/application_for_certificate_of_pardon_for_simple_marijuana_possession.pdf' + ), + path: 'test.pdf', + fields: {}, + }), + }, + { + pattern: new PackageDownload({ text: 'Download now!' }).toPattern(), + session, + data: {}, + } + ); expect(result).toEqual( expect.objectContaining({ success: true, @@ -83,8 +103,8 @@ const createTestForm = async (): Promise => { }, outputs: [ { + id: 'test-id', path: 'test.pdf', - data: new Uint8Array(pdfBytes), fields: { [input1.id]: { type: 'TextField', diff --git a/packages/forms/src/patterns/package-download/submit.ts b/packages/forms/src/patterns/package-download/submit.ts index ad240a71..462db543 100644 --- a/packages/forms/src/patterns/package-download/submit.ts +++ b/packages/forms/src/patterns/package-download/submit.ts @@ -1,6 +1,6 @@ -import { failure, success } from '@atj/common'; +import { failure, success, type Result } from '@atj/common'; -import { type Blueprint } from '../..'; +import { type Blueprint, type FormOutput } from '../..'; import { createFormOutputFieldData, fillPDF } from '../../documents'; import { sessionIsComplete } from '../../session'; import { type SubmitHandler } from '../../submission'; @@ -9,17 +9,38 @@ import { type PackageDownloadPattern } from './index'; export const downloadPackageHandler: SubmitHandler< PackageDownloadPattern -> = async (config, opts) => { - if (!sessionIsComplete(config, opts.session)) { +> = async (context, opts) => { + if (!sessionIsComplete(context.config, opts.session)) { return failure('Form is not complete'); } + const outputsResult: Result<(FormOutput & { data: Uint8Array })[]> = + await Promise.all( + opts.session.form.outputs.map(async output => { + const doc = await context.getDocument(output.id); + if (!doc.success) { + throw new Error(doc.error); + } + return { + id: output.id, + path: doc.data.path, + fields: output.fields, + formFields: output.formFields, + data: doc.data.data, + }; + }) + ) + .then(values => success(values)) + .catch(error => failure(error)); + if (!outputsResult.success) { + return failure(outputsResult.error); + } + const documentsResult = await generateDocumentPackage( - opts.session.form, + outputsResult.data, opts.session.data.values ); if (!documentsResult.success) { - console.log('values', opts.session.data.values); return failure(documentsResult.error); } @@ -30,12 +51,12 @@ export const downloadPackageHandler: SubmitHandler< }; const generateDocumentPackage = async ( - form: Blueprint, + outputs: (FormOutput & { data: Uint8Array })[], formData: Record ) => { const errors = new Array(); const documents = new Array<{ fileName: string; data: Uint8Array }>(); - for (const document of form.outputs) { + for (const document of outputs) { const docFieldData = createFormOutputFieldData(document, formData); const pdfDocument = await fillPDF(document.data, docFieldData); if (!pdfDocument.success) { diff --git a/packages/forms/src/patterns/page-set/submit.test.ts b/packages/forms/src/patterns/page-set/submit.test.ts index 0cd041ed..bd186e7b 100644 --- a/packages/forms/src/patterns/page-set/submit.test.ts +++ b/packages/forms/src/patterns/page-set/submit.test.ts @@ -7,17 +7,27 @@ import { createFormSession } from '../../session'; import { PageSet } from './builder'; import { submitPage } from './submit'; +import { success } from '@atj/common'; describe('Page-set submission', () => { it('stores session data for valid page data', async () => { const session = createTestSession(); - const result = await submitPage(defaultFormConfig, { - pattern: session.form.patterns['page-set-1'], - session, - data: { - 'input-1': 'test', + const result = await submitPage( + { + config: defaultFormConfig, + getDocument: () => + Promise.resolve( + success({ id: 'id', data: new Uint8Array(), path: '', fields: {} }) + ), }, - }); + { + pattern: session.form.patterns['page-set-1'], + session, + data: { + 'input-1': 'test', + }, + } + ); expect(result).toEqual({ data: { session: { @@ -43,13 +53,22 @@ describe('Page-set submission', () => { it('stores session data for invalid page data', async () => { const session = createTestSession(); - const result = await submitPage(defaultFormConfig, { - pattern: session.form.patterns['page-set-1'], - session, - data: { - 'input-1': '', + const result = await submitPage( + { + config: defaultFormConfig, + getDocument: () => + Promise.resolve( + success({ id: 'id', data: new Uint8Array(), path: '', fields: {} }) + ), }, - }); + { + pattern: session.form.patterns['page-set-1'], + session, + data: { + 'input-1': '', + }, + } + ); expect(result).toEqual({ data: { session: { @@ -80,21 +99,30 @@ describe('Page-set submission', () => { it('terminates on the last page', async () => { const session = createTestSession(); - const result = await submitPage(defaultFormConfig, { - pattern: session.form.patterns['page-set-1'], - session: { - ...session, - route: { - url: '#', - params: { - page: '1', + const result = await submitPage( + { + config: defaultFormConfig, + getDocument: () => + Promise.resolve( + success({ id: 'id', data: new Uint8Array(), path: '', fields: {} }) + ), + }, + { + pattern: session.form.patterns['page-set-1'], + session: { + ...session, + route: { + url: '#', + params: { + page: '1', + }, }, }, - }, - data: { - 'input-2': 'test', - }, - }); + data: { + 'input-2': 'test', + }, + } + ); expect(result).toEqual({ data: { session: { diff --git a/packages/forms/src/patterns/page-set/submit.ts b/packages/forms/src/patterns/page-set/submit.ts index 3aa22ae4..e6e2aee6 100644 --- a/packages/forms/src/patterns/page-set/submit.ts +++ b/packages/forms/src/patterns/page-set/submit.ts @@ -16,7 +16,7 @@ const getPage = (formSession: FormSession) => { }; export const submitPage: SubmitHandler = async ( - config, + context, opts ) => { const pageNumber = getPage(opts.session); @@ -25,7 +25,7 @@ export const submitPage: SubmitHandler = async ( return failure(`Page ${pageNumber} does not exist`); } - const pagePatternConfig = getPatternConfig(config, 'page'); + const pagePatternConfig = getPatternConfig(context.config, 'page'); const pagePattern = getPatternSafely({ type: 'page', form: opts.session.form, @@ -36,7 +36,7 @@ export const submitPage: SubmitHandler = async ( } const result = aggregatePatternSessionValues( - config, + context.config, opts.session.form, pagePatternConfig, pagePattern.data, diff --git a/packages/forms/src/repository/add-document.test.ts b/packages/forms/src/repository/add-document.test.ts new file mode 100644 index 00000000..6786d16c --- /dev/null +++ b/packages/forms/src/repository/add-document.test.ts @@ -0,0 +1,33 @@ +import { beforeAll, expect, it, vi } from 'vitest'; + +import { type DbTestContext, describeDatabase } from '@atj/database/testing'; +import { addDocument } from './add-document.js'; +import type { ParsedPdf } from '../documents/pdf/parsing-api.js'; +import type { DocumentFieldMap } from '../documents/types.js'; +import { defaultFormConfig } from '../patterns/index.js'; + +describeDatabase('add document', () => { + const today = new Date(2000, 1, 1); + + beforeAll(async () => { + vi.setSystemTime(today); + }); + + it('works', async ({ db }) => { + const result = await addDocument( + { db: db.ctx, formConfig: defaultFormConfig }, + { + fileName: 'file.pdf', + data: new Uint8Array([1, 2, 3]), + extract: { + parsedPdf: {} as ParsedPdf, + fields: {} as DocumentFieldMap, + }, + } + ); + if (result.success === false) { + expect.fail(`addDocument failed: ${result.error}`); + } + expect(result.data.id).toBeTypeOf('string'); + }); +}); diff --git a/packages/forms/src/repository/add-document.ts b/packages/forms/src/repository/add-document.ts new file mode 100644 index 00000000..7945cb6e --- /dev/null +++ b/packages/forms/src/repository/add-document.ts @@ -0,0 +1,39 @@ +import { type Result, failure, success } from '@atj/common'; + +import type { ParsedPdf } from '../documents/pdf/parsing-api'; +import type { DocumentFieldMap } from '../documents/types'; +import type { FormRepositoryContext } from '.'; + +export type AddDocument = ( + ctx: FormRepositoryContext, + document: { + fileName: string; + data: Uint8Array; + extract: { + parsedPdf: ParsedPdf; + fields: DocumentFieldMap; + }; + } +) => Promise>; + +export const addDocument: AddDocument = async (ctx, document) => { + const uuid = crypto.randomUUID(); + const db = await ctx.db.getKysely(); + + return await db + .insertInto('form_documents') + .values({ + id: uuid, + type: 'pdf', + file_name: document.fileName, + data: Buffer.from(document.data), + extract: JSON.stringify(document.extract), + }) + .execute() + .then(() => + success({ + id: uuid, + }) + ) + .catch(err => failure(err.message)); +}; diff --git a/packages/forms/src/repository/add-form.test.ts b/packages/forms/src/repository/add-form.test.ts index 17f8b5c7..1e815b68 100644 --- a/packages/forms/src/repository/add-form.test.ts +++ b/packages/forms/src/repository/add-form.test.ts @@ -2,6 +2,7 @@ import { beforeAll, expect, it, vi } from 'vitest'; import { type DbTestContext, describeDatabase } from '@atj/database/testing'; import { addForm } from './add-form.js'; +import { defaultFormConfig } from '../patterns/index.js'; describeDatabase('add form', () => { const today = new Date(2000, 1, 1); @@ -11,7 +12,10 @@ describeDatabase('add form', () => { }); it('works', async ({ db }) => { - const result = await addForm(db.ctx, testForm); + const result = await addForm( + { db: db.ctx, formConfig: defaultFormConfig }, + testForm + ); if (result.success === false) { expect.fail('addForm failed'); } diff --git a/packages/forms/src/repository/add-form.ts b/packages/forms/src/repository/add-form.ts index 34370752..c272f2c7 100644 --- a/packages/forms/src/repository/add-form.ts +++ b/packages/forms/src/repository/add-form.ts @@ -1,22 +1,21 @@ import { type Result, failure, success } from '@atj/common'; -import { type DatabaseContext } from '@atj/database'; import { type Blueprint } from '../index.js'; -import { stringifyForm } from './serialize.js'; +import type { FormRepositoryContext } from './index.js'; export type AddForm = ( - ctx: DatabaseContext, + ctx: FormRepositoryContext, form: Blueprint ) => Promise>; export const addForm: AddForm = async (ctx, form) => { const uuid = crypto.randomUUID(); - const db = await ctx.getKysely(); + const db = await ctx.db.getKysely(); return db .insertInto('forms') .values({ id: uuid, - data: stringifyForm(form), + data: JSON.stringify(form), }) .execute() .then(() => diff --git a/packages/forms/src/repository/delete-form.test.ts b/packages/forms/src/repository/delete-form.test.ts index 5b104bb6..b1519162 100644 --- a/packages/forms/src/repository/delete-form.test.ts +++ b/packages/forms/src/repository/delete-form.test.ts @@ -1,7 +1,15 @@ import { beforeAll, expect, it, vi } from 'vitest'; +import type { Result } from '@atj/common'; import { type DbTestContext, describeDatabase } from '@atj/database/testing'; + +import { createTestBlueprint } from '../builder/builder.test.js'; +import type { Blueprint } from '../types.js'; + +import { addDocument } from './add-document.js'; +import { addForm } from './add-form.js'; import { deleteForm } from './delete-form.js'; +import { defaultFormConfig } from '../patterns/index.js'; describeDatabase('delete form', () => { const today = new Date(2000, 1, 1); @@ -21,11 +29,11 @@ describeDatabase('delete form', () => { .execute(); const result = await deleteForm( - db.ctx, + { db: db.ctx, formConfig: defaultFormConfig }, '45c66187-64e2-4d75-a45a-e80f1d035bc5' ); if (result.success === false) { - expect.fail('addForm failed'); + expect.fail(`addForm failed: ${result.error}`); } const selectResult = await kysely @@ -40,9 +48,123 @@ describeDatabase('delete form', () => { it('fails with invalid form ID', async ({ db }) => { const result = await deleteForm( - db.ctx, + { db: db.ctx, formConfig: defaultFormConfig }, '45c66187-64e2-4d75-a45a-e80f1d035bc5' ); expect(result.success).toBe(false); }); + + it('removes associated documents', async ({ db }) => { + // Setup + const { id: id1 } = await ensure( + await addDocument( + { db: db.ctx, formConfig: defaultFormConfig }, + { + fileName: 'test1.pdf', + data: new Uint8Array(), + extract: { + parsedPdf: { + patterns: {}, + errors: [], + outputs: {}, + root: 'root', + title: 'Test form', + description: 'Test description', + }, + fields: {}, + }, + } + ), + 'Failed to add test document' + ); + const { id: id2 } = await ensure( + await addDocument( + { db: db.ctx, formConfig: defaultFormConfig }, + { + fileName: 'test2.pdf', + data: new Uint8Array(), + extract: { + parsedPdf: { + patterns: {}, + errors: [], + outputs: {}, + root: 'root', + title: 'Test form', + description: 'Test description', + }, + fields: {}, + }, + } + ), + 'Failed to add test document' + ); + const form = createTestBlueprint(); + const { id: formId } = await ensure( + await addForm( + { db: db.ctx, formConfig: defaultFormConfig }, + { + ...form, + outputs: [ + { id: id1, path: 'test1.pdf', fields: {}, formFields: {} }, + { id: id2, path: 'test2.pdf', fields: {}, formFields: {} }, + ], + } + ), + 'Failed to add test form' + ); + + // Test + const result = await deleteForm( + { db: db.ctx, formConfig: defaultFormConfig }, + formId + ); + + // Assert + expect(result).toEqual({ success: true }); + // Select the count of rows in the form_documents table: + const kysely = await db.ctx.getKysely(); + const selectResult = await kysely + .selectFrom('form_documents') + .select(kysely.fn.count('id').as('count')) + .executeTakeFirst(); + expect(Number(selectResult?.count)).toEqual(0); + }); }); + +const ensure = (result: Result, message: string = 'Ensure failure') => { + if (result.success === false) { + expect.fail(`${message}: ${result.error}`); + } + return result.data; +}; + +const TEST_FORM: Blueprint = { + summary: { + title: 'Test form', + description: 'Test description', + }, + root: 'root', + patterns: { + root: { + type: 'sequence', + id: 'root', + data: { + patterns: [], + }, + }, + }, + outputs: [ + { + id: '1', + path: 'test1.pdf', + fields: {}, + formFields: {}, + }, + { + id: '2', + path: 'test2.pdf', + fields: {}, + formFields: {}, + }, + ], +}; diff --git a/packages/forms/src/repository/delete-form.ts b/packages/forms/src/repository/delete-form.ts index 0a78c835..022b5539 100644 --- a/packages/forms/src/repository/delete-form.ts +++ b/packages/forms/src/repository/delete-form.ts @@ -1,23 +1,45 @@ -import { type VoidResult, failure } from '@atj/common'; +import { type VoidResult, failure, voidSuccess } from '@atj/common'; -import { type DatabaseContext } from '@atj/database'; +import type { FormOutput } from '../types'; +import type { FormRepositoryContext } from '.'; export type DeleteForm = ( - ctx: DatabaseContext, + ctx: FormRepositoryContext, formId: string -) => Promise; +) => Promise>; export const deleteForm: DeleteForm = async (ctx, formId) => { - const db = await ctx.getKysely(); + const db = await ctx.db.getKysely(); - const deleteResult = await db - .deleteFrom('forms') - .where('id', '=', formId) - .execute(); + const result = await db.transaction().execute(async trx => { + const deleteResult = await trx + .deleteFrom('forms') + .where('id', '=', formId) + .returning('data') + .executeTakeFirst(); - if (!deleteResult[0].numDeletedRows) { - return failure('form not found'); - } + if (!deleteResult) { + return failure({ message: 'form not found', code: 'not-found' as const }); + } - return { success: true }; + const form = JSON.parse(deleteResult.data); + const documentIds: string[] = form.outputs.map( + (output: FormOutput) => output.id + ); + + if (documentIds.length === 0) { + return voidSuccess; + } + + return await trx + .deleteFrom('form_documents') + .where('id', 'in', documentIds) + .execute() + .then(_ => voidSuccess) + .catch((error: Error) => { + return failure({ message: error.message, code: 'unknown' as const }); + }); + }); + + return result; }; diff --git a/packages/forms/src/repository/get-document.test.ts b/packages/forms/src/repository/get-document.test.ts new file mode 100644 index 00000000..eb784ea8 --- /dev/null +++ b/packages/forms/src/repository/get-document.test.ts @@ -0,0 +1,45 @@ +import { beforeAll, expect, it, vi } from 'vitest'; + +import { type DbTestContext, describeDatabase } from '@atj/database/testing'; +import { addDocument } from './add-document'; +import type { DocumentFieldMap } from '../documents/types'; +import type { ParsedPdf } from '../documents/pdf/parsing-api'; +import { defaultFormConfig } from '../patterns'; +import { getDocument } from './get-document'; + +describeDatabase('get document', () => { + const today = new Date(2000, 1, 1); + + beforeAll(async () => { + vi.setSystemTime(today); + }); + + it('works', async ({ db }) => { + const context = { db: db.ctx, formConfig: defaultFormConfig }; + // add test document + const testDocument = { + fileName: 'file.pdf', + data: new Uint8Array([1, 2, 3]), + extract: { + parsedPdf: {} as ParsedPdf, + fields: {} as DocumentFieldMap, + }, + }; + const result = await addDocument(context, testDocument); + if (!result.success) { + expect.fail(`addDocument failed: ${result.error}`); + } + + // get the document + const docResult = await getDocument(context, result.data.id); + expect(docResult).toEqual({ + success: true, + data: { + id: result.data.id, + path: testDocument.fileName, + data: new Buffer(testDocument.data), + fields: {}, + }, + }); + }); +}); diff --git a/packages/forms/src/repository/get-document.ts b/packages/forms/src/repository/get-document.ts new file mode 100644 index 00000000..35b6c028 --- /dev/null +++ b/packages/forms/src/repository/get-document.ts @@ -0,0 +1,39 @@ +import { type Result, failure, success } from '@atj/common'; + +import type { ParsedPdf } from '../documents/pdf/parsing-api'; +import type { DocumentFieldMap } from '../documents/types'; +import type { FormRepositoryContext } from '.'; + +export type GetDocument = ( + ctx: FormRepositoryContext, + id: string +) => Promise< + Result<{ + id: string; + data: Uint8Array; + path: string; + fields: DocumentFieldMap; + //formFields: Record; + }> +>; + +export const getDocument: GetDocument = async (ctx, id) => { + const db = await ctx.db.getKysely(); + + return await db + .selectFrom('form_documents') + .select(['id', 'type', 'file_name', 'data', 'extract']) + .where('id', '=', id) + .executeTakeFirstOrThrow() + .then(data => { + const extract: { parsedPdf: ParsedPdf; fields: DocumentFieldMap } = + JSON.parse(data.extract); + return success({ + id: data.id, + data: data.data, + path: data.file_name, + fields: extract.fields, + }); + }) + .catch(err => failure(err.message)); +}; diff --git a/packages/forms/src/repository/get-form-list.test.ts b/packages/forms/src/repository/get-form-list.test.ts index 33841208..001704e3 100644 --- a/packages/forms/src/repository/get-form-list.test.ts +++ b/packages/forms/src/repository/get-form-list.test.ts @@ -3,9 +3,11 @@ import { expect, it } from 'vitest'; import { type DbTestContext, describeDatabase } from '@atj/database/testing'; import { getForm } from './get-form.js'; import { getFormList } from './get-form-list.js'; +import { defaultFormConfig } from '../patterns/index.js'; describeDatabase('getFormList', () => { it('retrieves form list successfully', async ({ db }) => { + const context = { db: db.ctx, formConfig: defaultFormConfig }; const kysely = await db.ctx.getKysely(); await kysely .insertInto('forms') @@ -21,7 +23,7 @@ describeDatabase('getFormList', () => { ]) .execute(); - const result = await getFormList(db.ctx); + const result = await getFormList(context); expect(result).toEqual([ { id: '45c66187-64e2-4d75-a45a-e80f1d035bc5', @@ -37,10 +39,11 @@ describeDatabase('getFormList', () => { }); it('return null with non-existent form', async ({ db }) => { + const context = { db: db.ctx, formConfig: defaultFormConfig }; const result = await getForm( - db.ctx, + context, '45c66187-64e2-4d75-a45a-e80f1d035bc5' ); - expect(result).toBeNull(); + expect(result).toEqual({ success: true, data: null }); }); }); diff --git a/packages/forms/src/repository/get-form-list.ts b/packages/forms/src/repository/get-form-list.ts index dc7ef3da..2f25fc7f 100644 --- a/packages/forms/src/repository/get-form-list.ts +++ b/packages/forms/src/repository/get-form-list.ts @@ -1,6 +1,6 @@ -import { type DatabaseContext } from '@atj/database'; +import type { FormRepositoryContext } from '.'; -export type GetFormList = (ctx: DatabaseContext) => Promise< +export type GetFormList = (ctx: FormRepositoryContext) => Promise< | { id: string; title: string; @@ -10,7 +10,7 @@ export type GetFormList = (ctx: DatabaseContext) => Promise< >; export const getFormList: GetFormList = async ctx => { - const db = await ctx.getKysely(); + const db = await ctx.db.getKysely(); const rows = await db.selectFrom('forms').select(['id', 'data']).execute(); return rows.map(row => { diff --git a/packages/forms/src/repository/get-form-session.test.ts b/packages/forms/src/repository/get-form-session.test.ts index f5e4c088..76cd350a 100644 --- a/packages/forms/src/repository/get-form-session.test.ts +++ b/packages/forms/src/repository/get-form-session.test.ts @@ -5,11 +5,13 @@ import { type DbTestContext, describeDatabase } from '@atj/database/testing'; import { createTestBlueprint } from '../builder/builder.test'; import { addForm } from './add-form'; import { getFormSession } from './get-form-session'; +import { defaultFormConfig } from '../patterns'; describeDatabase('getFormSession', () => { it('returns a preexisting form session', async ctx => { const db = await ctx.db.ctx.getKysely(); - const form = await addForm(ctx.db.ctx, createTestBlueprint()); + const context = { db: ctx.db.ctx, formConfig: defaultFormConfig }; + const form = await addForm(context, createTestBlueprint()); if (!form.success) { expect.fail(form.error); } @@ -23,7 +25,7 @@ describeDatabase('getFormSession', () => { }) .executeTakeFirstOrThrow(); - const formSessionResult = await getFormSession(ctx.db.ctx, formSessionId); + const formSessionResult = await getFormSession(context, formSessionId); if (!formSessionResult.success) { expect.fail(formSessionResult.error); } @@ -36,8 +38,9 @@ describeDatabase('getFormSession', () => { }); it('returns an error if the form session does not exist', async ctx => { + const context = { db: ctx.db.ctx, formConfig: defaultFormConfig }; const formSessionResult = await getFormSession( - ctx.db.ctx, + context, '7128b29f-e03d-48c8-8a82-2af8759fc146' ); expect(formSessionResult).toEqual({ diff --git a/packages/forms/src/repository/get-form-session.ts b/packages/forms/src/repository/get-form-session.ts index a6d4a4fe..d2aa7af7 100644 --- a/packages/forms/src/repository/get-form-session.ts +++ b/packages/forms/src/repository/get-form-session.ts @@ -1,9 +1,9 @@ import { type Result, failure, success } from '@atj/common'; -import { type DatabaseContext } from '@atj/database'; import { type FormSession, type FormSessionId } from '../session'; +import type { FormRepositoryContext } from '.'; export type GetFormSession = ( - ctx: DatabaseContext, + ctx: FormRepositoryContext, id: string ) => Promise< Result<{ @@ -14,7 +14,7 @@ export type GetFormSession = ( >; export const getFormSession: GetFormSession = async (ctx, id) => { - const db = await ctx.getKysely(); + const db = await ctx.db.getKysely(); return await db .selectFrom('form_sessions') .where('id', '=', id) diff --git a/packages/forms/src/repository/get-form.test.ts b/packages/forms/src/repository/get-form.test.ts index 415f809c..5d4441b1 100644 --- a/packages/forms/src/repository/get-form.test.ts +++ b/packages/forms/src/repository/get-form.test.ts @@ -2,7 +2,7 @@ import { expect, it } from 'vitest'; import { type DbTestContext, describeDatabase } from '@atj/database/testing'; -import { type Blueprint } from '../index.js'; +import { defaultFormConfig, type Blueprint } from '../index.js'; import { getForm } from './get-form.js'; describeDatabase('getForm', () => { @@ -12,34 +12,33 @@ describeDatabase('getForm', () => { .insertInto('forms') .values({ id: '45c66187-64e2-4d75-a45a-e80f1d035bc5', - data: '{"summary":{"title":"Title","description":"Description"},"root":"root","patterns":{"root":{"type":"sequence","id":"root","data":{"patterns":[]}}},"outputs":[{"data":"AQID","path":"test.pdf","fields":{},"formFields":{}}]}', + data: '{"summary":{"title":"Title","description":"Description"},"root":"root","patterns":{"root":{"type":"page-set","id":"root","data":{"pages":[]}}},"outputs":[{"id":"test-id","path":"test.pdf","fields":{},"formFields":{}}]}', }) .execute(); const result = await getForm( - db.ctx, + { db: db.ctx, formConfig: defaultFormConfig }, '45c66187-64e2-4d75-a45a-e80f1d035bc5' ); - console.log(result); - expect(result).toEqual(TEST_FORM); + expect(result).toEqual({ success: true, data: TEST_FORM }); }); it('return null with non-existent form', async ({ db }) => { const result = await getForm( - db.ctx, + { db: db.ctx, formConfig: defaultFormConfig }, '45c66187-64e2-4d75-a45a-e80f1d035bc5' ); - expect(result).toBeNull(); + expect(result).toEqual({ success: true, data: null }); }); }); const TEST_FORM: Blueprint = { summary: { title: 'Title', description: 'Description' }, root: 'root', - patterns: { root: { type: 'sequence', id: 'root', data: { patterns: [] } } }, + patterns: { root: { type: 'page-set', id: 'root', data: { pages: [] } } }, outputs: [ { - data: new Uint8Array([1, 2, 3]), + id: 'test-id', path: 'test.pdf', fields: {}, formFields: {}, diff --git a/packages/forms/src/repository/get-form.ts b/packages/forms/src/repository/get-form.ts index e144162b..eeb11de2 100644 --- a/packages/forms/src/repository/get-form.ts +++ b/packages/forms/src/repository/get-form.ts @@ -1,14 +1,15 @@ -import { type DatabaseContext } from '@atj/database'; - +import { failure, success, type Result } from '@atj/common'; +import { parseFormString } from '../builder/parse-form.js'; import { type Blueprint } from '../index.js'; +import type { FormRepositoryContext } from './index.js'; export type GetForm = ( - ctx: DatabaseContext, + ctx: FormRepositoryContext, formId: string -) => Promise; +) => Promise>; export const getForm: GetForm = async (ctx, formId) => { - const db = await ctx.getKysely(); + const db = await ctx.db.getKysely(); const selectResult = await db .selectFrom('forms') .select(['data']) @@ -16,29 +17,13 @@ export const getForm: GetForm = async (ctx, formId) => { .executeTakeFirst(); if (selectResult === undefined) { - return null; + return success(null); } - return parseStringForm(selectResult.data); -}; - -const parseStringForm = (formString: string): Blueprint => { - const form = JSON.parse(formString) as Blueprint; - return { - ...form, - outputs: form.outputs.map((output: any) => ({ - ...output, - data: base64ToUint8Array((output as any).data), - })), - }; -}; - -const base64ToUint8Array = (base64: string): Uint8Array => { - const binaryString = atob(base64); - const len = binaryString.length; - const bytes = new Uint8Array(len); - for (let i = 0; i < len; i++) { - bytes[i] = binaryString.charCodeAt(i); + const parseResult = parseFormString(ctx.formConfig, selectResult.data); + if (!parseResult.success) { + return failure(`Failed to parse form: ${parseResult.error}`); } - return bytes; + + return success(parseResult.data); }; diff --git a/packages/forms/src/repository/index.ts b/packages/forms/src/repository/index.ts index dd469d4e..3c250ffc 100644 --- a/packages/forms/src/repository/index.ts +++ b/packages/forms/src/repository/index.ts @@ -1,8 +1,12 @@ import { type ServiceMethod, createService } from '@atj/common'; import { type DatabaseContext } from '@atj/database'; +import type { FormConfig } from '../pattern.js'; + +import { type AddDocument, addDocument } from './add-document.js'; import { type AddForm, addForm } from './add-form.js'; import { type DeleteForm, deleteForm } from './delete-form.js'; +import { type GetDocument, getDocument } from './get-document.js'; import { type GetForm, getForm } from './get-form.js'; import { type GetFormList, getFormList } from './get-form-list.js'; import { type GetFormSession, getFormSession } from './get-form-session.js'; @@ -13,8 +17,10 @@ import { } from './upsert-form-session.js'; export interface FormRepository { + addDocument: ServiceMethod; addForm: ServiceMethod; deleteForm: ServiceMethod; + getDocument: ServiceMethod; getForm: ServiceMethod; getFormSession: ServiceMethod; getFormList: ServiceMethod; @@ -22,10 +28,19 @@ export interface FormRepository { upsertFormSession: ServiceMethod; } -export const createFormsRepository = (ctx: DatabaseContext): FormRepository => +export type FormRepositoryContext = { + db: DatabaseContext; + formConfig: FormConfig; +}; + +export const createFormsRepository = ( + ctx: FormRepositoryContext +): FormRepository => createService(ctx, { + addDocument, addForm, deleteForm, + getDocument, getFormList, getFormSession, getForm, diff --git a/packages/forms/src/repository/save-form.test.ts b/packages/forms/src/repository/save-form.test.ts index 8befac51..62c7f379 100644 --- a/packages/forms/src/repository/save-form.test.ts +++ b/packages/forms/src/repository/save-form.test.ts @@ -2,7 +2,7 @@ import { expect, it } from 'vitest'; import { type DbTestContext, describeDatabase } from '@atj/database/testing'; -import { type Blueprint } from '../index.js'; +import { defaultFormConfig, type Blueprint } from '../index.js'; import { saveForm } from './save-form.js'; import { addForm } from './add-form.js'; @@ -23,7 +23,7 @@ const TEST_FORM: Blueprint = { }, outputs: [ { - data: new Uint8Array([1, 2, 3]), + id: 'test-id', path: 'test.pdf', fields: {}, formFields: {}, @@ -34,15 +34,22 @@ const TEST_FORM: Blueprint = { describeDatabase('saveForm', () => { it('saves pre-existing form successfully', async ({ db }) => { const kysely = await db.ctx.getKysely(); - const addResult = await addForm(db.ctx, TEST_FORM); + const addResult = await addForm( + { db: db.ctx, formConfig: defaultFormConfig }, + TEST_FORM + ); if (!addResult.success) { throw new Error('Failed to add form'); } - const saveResult = await saveForm(db.ctx, addResult.data.id, { - ...TEST_FORM, - summary: { title: 'Updated title', description: 'Updated description' }, - }); + const saveResult = await saveForm( + { db: db.ctx, formConfig: defaultFormConfig }, + addResult.data.id, + { + ...TEST_FORM, + summary: { title: 'Updated title', description: 'Updated description' }, + } + ); if (!saveResult.success) { expect.fail('Failed to save form', saveResult.error); } @@ -55,7 +62,7 @@ describeDatabase('saveForm', () => { expect(result[0].id).toEqual(addResult.data.id); expect(result[0].data).toEqual( - '{"summary":{"title":"Updated title","description":"Updated description"},"root":"root","patterns":{"root":{"type":"sequence","id":"root","data":{"patterns":[]}}},"outputs":[{"data":"AQID","path":"test.pdf","fields":{},"formFields":{}}]}' + '{"summary":{"title":"Updated title","description":"Updated description"},"root":"root","patterns":{"root":{"type":"sequence","id":"root","data":{"patterns":[]}}},"outputs":[{"id":"test-id","path":"test.pdf","fields":{},"formFields":{}}]}' ); }); }); diff --git a/packages/forms/src/repository/save-form.ts b/packages/forms/src/repository/save-form.ts index 18365c80..438f5d65 100644 --- a/packages/forms/src/repository/save-form.ts +++ b/packages/forms/src/repository/save-form.ts @@ -1,22 +1,21 @@ import { type VoidResult, failure, success } from '@atj/common'; -import { type DatabaseContext } from '@atj/database'; import { type Blueprint } from '../index.js'; -import { stringifyForm } from './serialize.js'; +import type { FormRepositoryContext } from './index.js'; export type SaveForm = ( - ctx: DatabaseContext, + ctx: FormRepositoryContext, formId: string, form: Blueprint ) => Promise; export const saveForm: SaveForm = async (ctx, id, blueprint) => { - const db = await ctx.getKysely(); + const db = await ctx.db.getKysely(); return await db .updateTable('forms') .set({ - data: stringifyForm(blueprint), + data: JSON.stringify(blueprint), }) .where('id', '=', id) .execute() diff --git a/packages/forms/src/repository/serialize.ts b/packages/forms/src/repository/serialize.ts deleted file mode 100644 index 11e32283..00000000 --- a/packages/forms/src/repository/serialize.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { type Blueprint } from '..'; - -export const stringifyForm = (form: Blueprint) => { - return JSON.stringify({ - ...form, - outputs: form.outputs.map((output: any) => ({ - ...output, - // TODO: we probably want to do this somewhere in the documents module - data: uint8ArrayToBase64(output.data), - })), - }); -}; - -const uint8ArrayToBase64 = (buffer: Uint8Array): string => { - let binary = ''; - const len = buffer.byteLength; - for (let i = 0; i < len; i++) { - binary += String.fromCharCode(buffer[i]); - } - return btoa(binary); -}; diff --git a/packages/forms/src/repository/upsert-form-session.test.ts b/packages/forms/src/repository/upsert-form-session.test.ts index 612a7d35..26d18997 100644 --- a/packages/forms/src/repository/upsert-form-session.test.ts +++ b/packages/forms/src/repository/upsert-form-session.test.ts @@ -2,7 +2,7 @@ import { beforeEach, expect, it } from 'vitest'; import { type DbTestContext, describeDatabase } from '@atj/database/testing'; -import { type Blueprint } from '..'; +import { defaultFormConfig, type Blueprint } from '..'; import { createTestBlueprint } from '../builder/builder.test'; import { type FormSession } from '../session'; @@ -18,7 +18,10 @@ type UpsertTestContext = DbTestContext & { describeDatabase('upsertFormSession', () => { beforeEach(async ctx => { ctx.form = createTestBlueprint(); - const addFormResult = await addForm(ctx.db.ctx, ctx.form); + const addFormResult = await addForm( + { db: ctx.db.ctx, formConfig: defaultFormConfig }, + ctx.form + ); if (!addFormResult.success) { expect.fail('Failed to add test form'); } @@ -31,10 +34,13 @@ describeDatabase('upsertFormSession', () => { }); it('creates and updates form session', async ctx => { - const result = await upsertFormSession(ctx.db.ctx, { - formId: ctx.formId, - data: ctx.sessionData, - }); + const result = await upsertFormSession( + { db: ctx.db.ctx, formConfig: defaultFormConfig }, + { + formId: ctx.formId, + data: ctx.sessionData, + } + ); if (!result.success) { expect.fail(result.error); } @@ -57,7 +63,10 @@ describeDatabase('upsertFormSession', () => { form: ctx.form, }, }; - const result2 = await upsertFormSession(ctx.db.ctx, formSession); + const result2 = await upsertFormSession( + { db: ctx.db.ctx, formConfig: defaultFormConfig }, + formSession + ); if (!result2.success) { expect.fail(result2.error); } diff --git a/packages/forms/src/repository/upsert-form-session.ts b/packages/forms/src/repository/upsert-form-session.ts index 2d21edb0..830f71d7 100644 --- a/packages/forms/src/repository/upsert-form-session.ts +++ b/packages/forms/src/repository/upsert-form-session.ts @@ -1,9 +1,9 @@ import { type Result, failure, success } from '@atj/common'; -import { type DatabaseContext } from '@atj/database'; import { type FormSession } from '../session'; +import type { FormRepositoryContext } from '.'; export type UpsertFormSession = ( - ctx: DatabaseContext, + ctx: FormRepositoryContext, opts: { id?: string; formId: string; @@ -12,7 +12,7 @@ export type UpsertFormSession = ( ) => Promise>; export const upsertFormSession: UpsertFormSession = async (ctx, opts) => { - const db = await ctx.getKysely(); + const db = await ctx.db.getKysely(); const strData = JSON.stringify(opts.data); const id = opts.id || crypto.randomUUID(); return await db diff --git a/packages/forms/src/services/delete-form.ts b/packages/forms/src/services/delete-form.ts index a4aaa40b..f3eb1b60 100644 --- a/packages/forms/src/services/delete-form.ts +++ b/packages/forms/src/services/delete-form.ts @@ -19,8 +19,14 @@ export const deleteForm: DeleteForm = async (ctx, formId) => { message: 'You must be logged in to delete a form', }); } - const form = await ctx.repository.getForm(formId); - if (form === null) { + const formResult = await ctx.repository.getForm(formId); + if (!formResult.success) { + return failure({ + status: 500, + message: formResult.error, + }); + } + if (formResult.data === null) { return failure({ status: 404, message: `form '${formId} does not exist`, diff --git a/packages/forms/src/services/get-form-session.ts b/packages/forms/src/services/get-form-session.ts index eb3521af..13767d4d 100644 --- a/packages/forms/src/services/get-form-session.ts +++ b/packages/forms/src/services/get-form-session.ts @@ -24,15 +24,22 @@ export type GetFormSession = ( >; export const getFormSession: GetFormSession = async (ctx, opts) => { - const form = await ctx.repository.getForm(opts.formId); - if (form === null) { + const formResult = await ctx.repository.getForm(opts.formId); + if (!formResult.success) { + return failure(`Failed to retrieve form: ${formResult.error}`); + } + + if (formResult.data === null) { return failure(`form '${opts.formId} does not exist`); } // If this request corresponds to an non-existent session, return a new // session that is not yet persisted. if (opts.sessionId === undefined) { - const formSession = await createFormSession(form, opts.formRoute); + const formSession = await createFormSession( + formResult.data, + opts.formRoute + ); return success({ formId: opts.formId, data: formSession, @@ -44,7 +51,7 @@ export const getFormSession: GetFormSession = async (ctx, opts) => { console.error( `Error retrieving form session: ${formSession.error}. Returning new session.` ); - const newSession = await createFormSession(form, opts.formRoute); + const newSession = await createFormSession(formResult.data, opts.formRoute); return success({ formId: opts.formId, data: newSession, diff --git a/packages/forms/src/services/get-form.test.ts b/packages/forms/src/services/get-form.test.ts index 412f10c0..be59cee7 100644 --- a/packages/forms/src/services/get-form.test.ts +++ b/packages/forms/src/services/get-form.test.ts @@ -33,7 +33,7 @@ describe('getForm', () => { const result = await getForm(ctx, addResult.data.id); if (!result.success) { - expect.fail('Failed to add form:', result.error); + expect.fail(`Failed to get form: ${JSON.stringify(result.error)}`); } expect(result.data).toEqual(TEST_FORM); }); diff --git a/packages/forms/src/services/get-form.ts b/packages/forms/src/services/get-form.ts index 82737020..d2c0f67a 100644 --- a/packages/forms/src/services/get-form.ts +++ b/packages/forms/src/services/get-form.ts @@ -1,7 +1,8 @@ import { type Result, failure, success } from '@atj/common'; -import { type Blueprint } from '../index.js'; +import { parseForm } from '../builder/parse-form.js'; import { type FormServiceContext } from '../context/index.js'; +import { type Blueprint } from '../types.js'; type GetFormError = { status: number; @@ -14,12 +15,20 @@ export type GetForm = ( ) => Promise>; export const getForm: GetForm = async (ctx, formId) => { - const result = await ctx.repository.getForm(formId); - if (result === null) { + const formResult = await ctx.repository.getForm(formId); + if (!formResult.success) { + return failure({ + status: 500, + message: formResult.error, + }); + } + + if (formResult.data === null) { return failure({ status: 404, message: 'Form not found', }); } - return success(result); + + return success(formResult.data); }; diff --git a/packages/forms/src/services/index.ts b/packages/forms/src/services/index.ts index 96cc598a..d426dc29 100644 --- a/packages/forms/src/services/index.ts +++ b/packages/forms/src/services/index.ts @@ -1,4 +1,4 @@ -import { createService, ServiceMethod } from '@atj/common'; +import { type ServiceMethod, createService } from '@atj/common'; import { type FormServiceContext } from '../context/index.js'; @@ -7,6 +7,7 @@ import { type DeleteForm, deleteForm } from './delete-form.js'; import { type GetForm, getForm } from './get-form.js'; import { type GetFormList, getFormList } from './get-form-list.js'; import { type GetFormSession, getFormSession } from './get-form-session.js'; +import { type InitializeForm, initializeForm } from './initialize-form.js'; import { type SaveForm, saveForm } from './save-form.js'; import { type SubmitForm, submitForm } from './submit-form.js'; @@ -17,6 +18,7 @@ export const createFormService = (ctx: FormServiceContext) => getForm, getFormList, getFormSession, + initializeForm, saveForm, submitForm, }); @@ -27,6 +29,7 @@ export type FormService = { getForm: ServiceMethod; getFormList: ServiceMethod; getFormSession: ServiceMethod; + initializeForm: ServiceMethod; saveForm: ServiceMethod; submitForm: ServiceMethod; }; diff --git a/packages/forms/src/services/initialize-form.test.ts b/packages/forms/src/services/initialize-form.test.ts new file mode 100644 index 00000000..a1c994ab --- /dev/null +++ b/packages/forms/src/services/initialize-form.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from 'vitest'; + +import { createTestFormServiceContext } from '../testing.js'; + +import { initializeForm } from './initialize-form.js'; + +const summary = { title: 'Form Title', description: '' }; + +describe('initializeForm', () => { + it('returns access denied (401) if user is not logged in', async () => { + const ctx = await createTestFormServiceContext({ + isUserLoggedIn: () => false, + }); + const result = await initializeForm(ctx, { summary }); + expect(result).toEqual({ + success: false, + error: { + status: 401, + message: 'You must be logged in to initialize a new form', + }, + }); + }); + + it('initializes with summary when user is logged in', async () => { + const ctx = await createTestFormServiceContext({ + isUserLoggedIn: () => true, + }); + const result = await initializeForm(ctx, { summary }); + expect(result).toEqual({ + success: true, + data: { + timestamp: expect.any(String), + id: expect.any(String), + }, + }); + }); + + it('initializes successfully with document when user is logged in', async () => { + const ctx = await createTestFormServiceContext({ + isUserLoggedIn: () => true, + parsedPdf: async () => ({ + parsedPdf: { + text: 'test', + title: '', + root: 'root', + description: '', + patterns: {}, + errors: [], + outputs: {}, + }, + fields: {}, + }), + }); + const result = await initializeForm(ctx, { + summary, + document: { fileName: 'test.pdf', data: 'VGhpcyBpcyBub3QgYSBQREYu' }, + }); + expect(result).toEqual({ + success: true, + data: { + timestamp: expect.any(String), + id: expect.any(String), + }, + }); + }); +}); diff --git a/packages/forms/src/services/initialize-form.ts b/packages/forms/src/services/initialize-form.ts new file mode 100644 index 00000000..218d1641 --- /dev/null +++ b/packages/forms/src/services/initialize-form.ts @@ -0,0 +1,121 @@ +import * as z from 'zod'; + +import { type Result, failure, success } from '@atj/common'; + +import { BlueprintBuilder } from '../builder/index.js'; +import { type FormServiceContext } from '../context/index.js'; +import type { FormSummary } from '../types.js'; +import { base64ToUint8Array } from '../util/base64.js'; + +type InitializeFormError = { + status: number; + message: string; +}; +type InitializeFormResult = { + timestamp: string; + id: string; +}; + +export type InitializeForm = ( + ctx: FormServiceContext, + opts: + | unknown + | { + summary?: FormSummary; + document?: { fileName: string; data: Uint8Array }; + } +) => Promise>; + +const base64 = + /^(?:[A-Za-z0-9+\/]{4})*(?:[A-Za-z0-9+\/]{2}==|[A-Za-z0-9+\/]{3}=)?$/; + +const optionSchema = z.object({ + summary: z + .object({ + title: z.string(), + description: z.string(), + }) + .optional(), + document: z + .object({ + fileName: z.string(), + data: z + .string() + .refine(value => base64.test(value), { + message: 'Invalid base64 string', + }) + .transform(value => base64ToUint8Array(value)), + }) + .optional(), +}); + +export const initializeForm: InitializeForm = async (ctx, opts) => { + if (!ctx.isUserLoggedIn()) { + return failure({ + status: 401, + message: 'You must be logged in to initialize a new form', + }); + } + + const parseResult = optionSchema.safeParse(opts); + if (!parseResult.success) { + return failure({ + status: 400, + message: 'Invalid options', + }); + } + const { document, summary } = parseResult.data; + + const builder = new BlueprintBuilder(ctx.config); + if (document !== undefined) { + const parsePdfResult = await ctx + .parsePdf(document.data) + .then(result => success(result)) + .catch(err => + failure({ + status: 400, + message: `Failed to parse PDF: ${err.message}`, + }) + ); + if (!parsePdfResult.success) { + return parsePdfResult; + } + const { parsedPdf } = parsePdfResult.data; + + builder.setFormSummary({ + title: parsedPdf.title || document.fileName, + description: parsedPdf.description, + }); + + const fileName = document.fileName.split('/').pop() || 'my-form.pdf'; + const addDocumentResult = await ctx.repository.addDocument({ + fileName, + data: document.data, + extract: parsePdfResult.data, + }); + if (!addDocumentResult.success) { + return failure({ + status: 500, + message: `Failed to add document: ${addDocumentResult.error}`, + }); + } + await builder.addDocumentRef({ + id: addDocumentResult.data.id, + extract: parsedPdf, + }); + } + + if (summary) { + builder.setFormSummary(summary); + } + + const result = await ctx.repository.addForm(builder.form); + if (!result.success) { + console.error('Failed to add form:', result.error); + return failure({ + status: 500, + message: result.error, + }); + } + return result; +}; diff --git a/packages/forms/src/services/save-form.test.ts b/packages/forms/src/services/save-form.test.ts index 94e3f873..3c180fef 100644 --- a/packages/forms/src/services/save-form.test.ts +++ b/packages/forms/src/services/save-form.test.ts @@ -4,6 +4,7 @@ import { createForm } from '../index.js'; import { createTestFormServiceContext } from '../testing.js'; import { saveForm } from './save-form.js'; +import { success } from '@atj/common'; const TEST_FORM = createForm({ title: 'Form Title', description: '' }); const TEST_FORM_2 = { @@ -42,9 +43,11 @@ describe('saveForm', () => { if (!result.success) { expect.fail('Failed to add form:', result.error); } - expect(result.data).toEqual({ timestamp: expect.any(Date) }); + expect(result.data).toEqual( + expect.objectContaining({ timestamp: expect.any(Date) }) + ); const savedForm = await ctx.repository.getForm(addResult.data.id); - expect(savedForm).toEqual(TEST_FORM_2); + expect(savedForm).toEqual(success(TEST_FORM_2)); }); }); diff --git a/packages/forms/src/services/save-form.ts b/packages/forms/src/services/save-form.ts index 133aed90..73d7245a 100644 --- a/packages/forms/src/services/save-form.ts +++ b/packages/forms/src/services/save-form.ts @@ -1,7 +1,8 @@ import { type Result, failure, success } from '@atj/common'; -import { Blueprint } from '../index.js'; import { type FormServiceContext } from '../context/index.js'; +import { type Blueprint } from '../types.js'; +import { parseForm } from '../builder/parse-form.js'; type SaveFormError = { status: number; @@ -21,7 +22,16 @@ export const saveForm: SaveForm = async (ctx, formId, form) => { message: 'You must be logged in to save a form', }); } - const result = await ctx.repository.saveForm(formId, form); + + const parseResult = parseForm(ctx.config, form); + if (!parseResult.success) { + return failure({ + status: 422, + message: parseResult.error, + }); + } + + const result = await ctx.repository.saveForm(formId, parseResult.data); if (result.success === false) { return failure({ status: 500, diff --git a/packages/forms/src/services/submit-form.ts b/packages/forms/src/services/submit-form.ts index ef917b79..533d1e6e 100644 --- a/packages/forms/src/services/submit-form.ts +++ b/packages/forms/src/services/submit-form.ts @@ -48,14 +48,18 @@ export const submitForm: SubmitForm = async ( formData, route ) => { - const form = await ctx.repository.getForm(formId); - if (form === null) { + const formResult = await ctx.repository.getForm(formId); + if (!formResult.success) { + return failure(formResult.error); + } + + if (formResult.data === null) { return failure('Form not found'); } const sessionResult = await getFormSessionOrCreate( ctx, - form, + formResult.data, route, sessionId ); @@ -75,17 +79,28 @@ export const submitForm: SubmitForm = async ( return failure(`Invalid action: ${actionString}`); } - const submitHandlerResult = registry.getHandlerForAction(form, actionString); + const submitHandlerResult = registry.getHandlerForAction( + formResult.data, + actionString + ); if (!submitHandlerResult.success) { return failure(submitHandlerResult.error); } const { handler, pattern } = submitHandlerResult.data; - const newSessionResult = await handler(ctx.config, { - pattern, - session, - data: formData, - }); + const newSessionResult = await handler( + { + config: ctx.config, + getDocument: id => { + return ctx.repository.getDocument(id); + }, + }, + { + pattern, + session, + data: formData, + } + ); if (!newSessionResult.success) { return failure(newSessionResult.error); diff --git a/packages/forms/src/session.ts b/packages/forms/src/session.ts index 76d5a49f..9823e034 100644 --- a/packages/forms/src/session.ts +++ b/packages/forms/src/session.ts @@ -141,7 +141,6 @@ export const updateSession = ( export const sessionIsComplete = (config: FormConfig, session: FormSession) => { return Object.values(session.form.patterns).every(pattern => { - console.log('validating', pattern.type, pattern.id); const patternConfig = getPatternConfig(config, pattern.type); const value = getFormSessionValue(session, pattern.id); const isValidResult = validatePattern(patternConfig, pattern, value); diff --git a/packages/forms/src/submission.ts b/packages/forms/src/submission.ts index f1f220b1..8b7b3a93 100644 --- a/packages/forms/src/submission.ts +++ b/packages/forms/src/submission.ts @@ -9,10 +9,22 @@ import { getPattern, } from './pattern'; import { type FormSession } from './session'; -import { type Blueprint } from '.'; +import { type Blueprint, type DocumentFieldMap } from '.'; + +export type SubmitHandlerContext = { + config: FormConfig; + getDocument: (id: string) => Promise< + Result<{ + id: string; + data: Uint8Array; + path: string; + fields: DocumentFieldMap; + }> + >; +}; export type SubmitHandler

= ( - config: FormConfig, + context: SubmitHandlerContext, opts: { pattern: P; session: FormSession; diff --git a/packages/forms/src/testing.ts b/packages/forms/src/testing.ts index cdad9203..a1fcc061 100644 --- a/packages/forms/src/testing.ts +++ b/packages/forms/src/testing.ts @@ -1,20 +1,29 @@ import { type DatabaseContext } from '@atj/database'; import { createInMemoryDatabaseContext } from '@atj/database/context'; -import { createFormsRepository } from './repository'; + +import type { FormServiceContext } from './context'; +import { type ParsePdf, parsePdf } from './documents'; import { defaultFormConfig } from './patterns'; +import { createFormsRepository } from './repository'; type Options = { isUserLoggedIn: () => boolean; + parsedPdf: ParsePdf; }; -export const createTestFormServiceContext = async (opts?: Partial) => { +export const createTestFormServiceContext = async ( + opts?: Partial +): Promise => { const db: DatabaseContext = await createInMemoryDatabaseContext(); - const repository = createFormsRepository(db); - return { + const repository = createFormsRepository({ db, + formConfig: defaultFormConfig, + }); + return { repository, config: defaultFormConfig, isUserLoggedIn: opts?.isUserLoggedIn || (() => true), + parsePdf: opts?.parsedPdf || parsePdf, }; }; diff --git a/packages/forms/src/types.ts b/packages/forms/src/types.ts index e5147eb4..d27d6bc9 100644 --- a/packages/forms/src/types.ts +++ b/packages/forms/src/types.ts @@ -14,7 +14,7 @@ export type FormSummary = { }; export type FormOutput = { - data: Uint8Array; + id: string; path: string; fields: DocumentFieldMap; formFields: Record; diff --git a/packages/forms/src/documents/util.ts b/packages/forms/src/util/base64.ts similarity index 65% rename from packages/forms/src/documents/util.ts rename to packages/forms/src/util/base64.ts index 90f6a49a..859a5746 100644 --- a/packages/forms/src/documents/util.ts +++ b/packages/forms/src/util/base64.ts @@ -12,7 +12,10 @@ export const stringToBase64 = (input: string): string => { } }; -export const uint8ArrayToBase64 = (uint8Array: Uint8Array) => { +export const uint8ArrayToBase64 = async (uint8Array: Uint8Array) => { + if (typeof Buffer !== 'undefined') { + return Buffer.from(uint8Array).toString('base64'); + } return new Promise((resolve, reject) => { const blob = new Blob([uint8Array], { type: 'application/octet-stream' }); const reader = new FileReader(); @@ -25,3 +28,13 @@ export const uint8ArrayToBase64 = (uint8Array: Uint8Array) => { reader.readAsDataURL(blob); }); }; + +export const base64ToUint8Array = (base64: string) => { + const binaryString = atob(base64); + const len = binaryString.length; + const bytes = new Uint8Array(len); + for (let i = 0; i < len; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + return bytes; +}; diff --git a/packages/server/src/config/services.ts b/packages/server/src/config/services.ts index a378a3b4..f0db284b 100644 --- a/packages/server/src/config/services.ts +++ b/packages/server/src/config/services.ts @@ -3,6 +3,7 @@ import { createFormService, createFormsRepository, defaultFormConfig, + parsePdf, } from '@atj/forms'; import { type ServerOptions } from './options.js'; @@ -11,8 +12,12 @@ export const createServerFormService = ( ctx: { isUserLoggedIn: () => boolean } ): FormService => { return createFormService({ - repository: createFormsRepository(options.db), + repository: createFormsRepository({ + db: options.db, + formConfig: defaultFormConfig, + }), config: defaultFormConfig, isUserLoggedIn: ctx.isUserLoggedIn, + parsePdf, }); }; diff --git a/packages/server/src/lib/api-client.ts b/packages/server/src/lib/api-client.ts index 374f79a1..74b70176 100644 --- a/packages/server/src/lib/api-client.ts +++ b/packages/server/src/lib/api-client.ts @@ -5,6 +5,7 @@ import { type FormSessionId, type Blueprint, type FormService, + type FormSummary, } from '@atj/forms'; import { type FormServiceContext } from '@atj/forms/context'; @@ -24,7 +25,43 @@ export class FormServiceClient implements FormService { }, }); const result = await response.json(); - console.log('addForm result', result); + return result; + } + + async initializeForm( + opts: + | unknown + | { + summary?: FormSummary; + document?: { fileName: string; data: string }; + } + ): Promise< + Result< + { timestamp: string; id: string }, + { status: number; message: string } + > + > { + const options = opts as { + summary?: FormSummary; + document?: { fileName: string; data: string }; + }; + const body = JSON.stringify({ + summary: options.summary ? options.summary : undefined, + document: options.document + ? { + fileName: options.document.fileName, + data: options.document.data, + } + : undefined, + }); + const response = await fetch(`${this.ctx.baseUrl}api/forms`, { + method: 'POST', + body, + headers: { + 'Content-Type': 'application/json', + }, + }); + const result = await response.json(); return result; } diff --git a/packages/server/src/lib/attachments.ts b/packages/server/src/lib/attachments.ts deleted file mode 100644 index 93a458c2..00000000 --- a/packages/server/src/lib/attachments.ts +++ /dev/null @@ -1,45 +0,0 @@ -export const createMultipartResponse = ( - pdfs: { fileName: string; data: Uint8Array }[] -): Response => { - const boundary = createBoundary(); - - // Array to store each part of the multipart message - const parts: Uint8Array[] = pdfs.flatMap(pdf => { - const headers = [ - `--${boundary}`, - `Content-Type: application/pdf`, - `Content-Disposition: attachment; filename="${pdf.fileName}"`, - '', - '', // empty line between headers and content - ].join('\r\n'); - - return [stringToUint8Array(headers), pdf.data, stringToUint8Array('\r\n')]; - }); - - // Final boundary to mark the end of the message - parts.push(stringToUint8Array(`--${boundary}--`)); - - // Concatenate all Uint8Array parts into a single Uint8Array body - const body = new Uint8Array( - parts.reduce((sum, part) => sum + part.length, 0) - ); - let offset = 0; - parts.forEach(part => { - body.set(part, offset); - offset += part.length; - }); - - return new Response(body, { - status: 200, - headers: { - 'Content-Type': `multipart/mixed; boundary=${boundary}`, - 'Content-Length': body.length.toString(), - }, - }); -}; - -const createBoundary = (): string => - `boundary_${Math.random().toString(36).slice(2)}`; - -const stringToUint8Array = (str: string): Uint8Array => - new TextEncoder().encode(str); diff --git a/packages/server/src/pages/api/forms/index.ts b/packages/server/src/pages/api/forms/index.ts index fe071985..15780209 100644 --- a/packages/server/src/pages/api/forms/index.ts +++ b/packages/server/src/pages/api/forms/index.ts @@ -13,9 +13,10 @@ export const GET: APIRoute = async context => { }; export const POST: APIRoute = async context => { - const form = await context.request.json(); + const input = await context.request.json(); const ctx = await getServerContext(context); - const result = await ctx.formService.addForm(form); + //const result = await ctx.formService.addForm(form); + const result = await ctx.formService.initializeForm(input); return new Response(JSON.stringify(result), { headers: { 'Content-Type': 'application/json', diff --git a/packages/server/src/pages/forms/[id].astro b/packages/server/src/pages/forms/[id].astro index 2a021bd6..69ab8059 100644 --- a/packages/server/src/pages/forms/[id].astro +++ b/packages/server/src/pages/forms/[id].astro @@ -70,7 +70,23 @@ if (Astro.request.method === 'POST') { setFormSessionCookie(submitFormResult.data.sessionId); if (submitFormResult.data.attachments) { - return createMultipartResponse(submitFormResult.data.attachments); + if (submitFormResult.data.attachments.length > 1) { + return new Response( + 'Multiple attachments are not supported at this time.', + { + status: 501, + } + ); + } + const attachment = submitFormResult.data.attachments[0]; + return new Response(attachment.data, { + headers: { + 'Content-Type': 'application/pdf', + 'Content-Disposition': `attachment; filename="${attachment.fileName}"`, + 'Content-Length': attachment.data.length.toString(), + }, + status: 200, + }); } return Astro.redirect(getNextUrl(submitFormResult.data.session.route));