Improve continuous export failure recovery by moving the export operation "progress marker" from USB drive to VxScan store

apps/central-scan/backend/src/store.ts

@@ -133,6 +133,10 @@ function dateTimeFromNoOffsetSqliteDate(noOffsetSqliteDate: string): DateTime {
 export class Store {
   private constructor(private readonly client: DbClient) {}
 
+  // Used by shared CVR export logic in libs/backend
+  // eslint-disable-next-line vx/gts-no-public-class-fields
+  readonly scannerType = 'central';
+
   getDbPath(): string {
     return this.client.getDatabasePath();
   }

apps/scan/backend/schema.sql

@@ -51,13 +51,19 @@ create table sheets (
 );
 
 create table system_settings (
-  -- enforce singleton table
+  -- Enforce singleton table
   id integer primary key check (id = 1),
   data text not null -- JSON blob
 );
 
+create table is_continuous_export_operation_in_progress (
+  -- Enforce singleton table
+  id integer primary key check (id = 1),
+  is_continuous_export_operation_in_progress boolean not null
+);
+
 create table export_directory_name (
-  -- enforce singleton table
+  -- Enforce singleton table
   id integer primary key check (id = 1),
   export_directory_name text not null
 );

apps/scan/backend/src/app_config.test.ts

@@ -164,9 +164,9 @@ test("if there's only one precinct in the election, it's selected automatically
 test('continuous CVR export', async () => {
   await withApp(
     {},
-    async ({ apiClient, mockAuth, mockScanner, mockUsbDrive }) => {
+    async ({ apiClient, mockAuth, mockScanner, mockUsbDrive, workspace }) => {
       await configureApp(apiClient, mockAuth, mockUsbDrive, { testMode: true });
-      await scanBallot(mockScanner, mockUsbDrive, apiClient, 0);
+      await scanBallot(mockScanner, apiClient, workspace.store, 0);
 
       const exportDirectoryPaths = await getCastVoteRecordExportDirectoryPaths(
         mockUsbDrive.usbDrive
@@ -216,10 +216,10 @@ test('continuous CVR export', async () => {
 test('continuous CVR export, including polls closing, followed by a full export', async () => {
   await withApp(
     {},
-    async ({ apiClient, mockAuth, mockScanner, mockUsbDrive }) => {
+    async ({ apiClient, mockAuth, mockScanner, mockUsbDrive, workspace }) => {
       await configureApp(apiClient, mockAuth, mockUsbDrive, { testMode: true });
-      await scanBallot(mockScanner, mockUsbDrive, apiClient, 0);
-      await scanBallot(mockScanner, mockUsbDrive, apiClient, 1);
+      await scanBallot(mockScanner, apiClient, workspace.store, 0);
+      await scanBallot(mockScanner, apiClient, workspace.store, 1);
 
       expect(
         await apiClient.exportCastVoteRecordsToUsbDrive({
@@ -239,9 +239,9 @@ test('continuous CVR export, including polls closing, followed by a full export'
 test('CVR resync', async () => {
   await withApp(
     {},
-    async ({ apiClient, mockAuth, mockScanner, mockUsbDrive }) => {
+    async ({ apiClient, mockAuth, mockScanner, mockUsbDrive, workspace }) => {
       await configureApp(apiClient, mockAuth, mockUsbDrive, { testMode: true });
-      await scanBallot(mockScanner, mockUsbDrive, apiClient, 0);
+      await scanBallot(mockScanner, apiClient, workspace.store, 0);
 
       // When a CVR resync is required, the CVR resync modal appears on the "insert your ballot"
       // screen, i.e. the screen displayed when no card is inserted
@@ -300,8 +300,8 @@ test('ballot batching', async () => {
       }
 
       // Scan two ballots, which should have the same batch
-      await scanBallot(mockScanner, mockUsbDrive, apiClient, 0);
-      await scanBallot(mockScanner, mockUsbDrive, apiClient, 1);
+      await scanBallot(mockScanner, apiClient, workspace.store, 0);
+      await scanBallot(mockScanner, apiClient, workspace.store, 1);
       let batchIds = getBatchIds();
       expect(getCvrIds()).toHaveLength(2);
       expect(batchIds).toHaveLength(1);
@@ -338,8 +338,8 @@ test('ballot batching', async () => {
       });
 
       // Confirm there is a new, second batch distinct from the first
-      await scanBallot(mockScanner, mockUsbDrive, apiClient, 2);
-      await scanBallot(mockScanner, mockUsbDrive, apiClient, 3);
+      await scanBallot(mockScanner, apiClient, workspace.store, 2);
+      await scanBallot(mockScanner, apiClient, workspace.store, 3);
       batchIds = getBatchIds();
       expect(getCvrIds()).toHaveLength(4);
       expect(batchIds).toHaveLength(2);
@@ -376,8 +376,8 @@ test('ballot batching', async () => {
       });
 
       // Confirm there is a third batch, distinct from the second
-      await scanBallot(mockScanner, mockUsbDrive, apiClient, 4);
-      await scanBallot(mockScanner, mockUsbDrive, apiClient, 5);
+      await scanBallot(mockScanner, apiClient, workspace.store, 4);
+      await scanBallot(mockScanner, apiClient, workspace.store, 5);
       batchIds = getBatchIds();
       expect(getCvrIds()).toHaveLength(6);
       expect(batchIds).toHaveLength(3);

apps/scan/backend/src/app_results.test.ts

@@ -28,13 +28,13 @@ beforeEach(() => {
 test('end-to-end tabulated results', async () => {
   await withApp(
     {},
-    async ({ apiClient, mockScanner, mockUsbDrive, mockAuth }) => {
+    async ({ apiClient, mockScanner, mockUsbDrive, mockAuth, workspace }) => {
       await configureApp(apiClient, mockAuth, mockUsbDrive, { testMode: true });
 
       // scan a few ballots
-      await scanBallot(mockScanner, mockUsbDrive, apiClient, 0);
-      await scanBallot(mockScanner, mockUsbDrive, apiClient, 1);
-      await scanBallot(mockScanner, mockUsbDrive, apiClient, 2);
+      await scanBallot(mockScanner, apiClient, workspace.store, 0);
+      await scanBallot(mockScanner, apiClient, workspace.store, 1);
+      await scanBallot(mockScanner, apiClient, workspace.store, 2);
 
       const allResults = await apiClient.getScannerResultsByParty();
       expect(allResults).toHaveLength(1);

apps/scan/backend/src/app_usb_drive.test.ts

@@ -46,7 +46,7 @@ test('ejectUsbDrive', async () => {
 test('doesUsbDriveRequireCastVoteRecordSync is properly populated', async () => {
   await withApp(
     {},
-    async ({ apiClient, mockAuth, mockUsbDrive, mockScanner }) => {
+    async ({ apiClient, mockAuth, mockUsbDrive, mockScanner, workspace }) => {
       await configureApp(apiClient, mockAuth, mockUsbDrive, { testMode: true });
       const mountedUsbDriveStatus = {
         status: 'mounted',
@@ -57,7 +57,7 @@ test('doesUsbDriveRequireCastVoteRecordSync is properly populated', async () =>
         mountedUsbDriveStatus
       );
 
-      await scanBallot(mockScanner, mockUsbDrive, apiClient, 0);
+      await scanBallot(mockScanner, apiClient, workspace.store, 0);
       await expect(apiClient.getUsbDriveStatus()).resolves.toEqual(
         mountedUsbDriveStatus
       );

apps/scan/backend/src/scanners/custom/state_machine.ts

@@ -434,11 +434,15 @@ async function recordAcceptedSheet(
   const { sheetId } = interpretation;
   store.withTransaction(() => {
     storeInterpretedSheet(store, sheetId, interpretation);
+
     // If we're storing an accepted sheet that needed review, that means that it was "adjudicated"
     // (i.e. the voter said to count it without changing anything).
     if (interpretation.type === 'NeedsReviewSheet') {
       store.adjudicateSheet(sheetId);
     }
+
+    // Gets reset to false within exportCastVoteRecordsToUsbDrive
+    store.setIsContinuousExportOperationInProgress(true);
   });
   (
     await exportCastVoteRecordsToUsbDrive(
@@ -460,10 +464,14 @@ async function recordRejectedSheet(
   const { sheetId } = interpretation;
   store.withTransaction(() => {
     storeInterpretedSheet(store, sheetId, interpretation);
+
     // We want to keep rejected ballots in the store, but not count them. We accomplish this by
     // "deleting" them, which just marks them as deleted and is how we indicate that an interpreted
     // ballot wasn't counted.
     store.deleteSheet(sheetId);
+
+    // Gets reset to false within exportCastVoteRecordsToUsbDrive
+    store.setIsContinuousExportOperationInProgress(true);
   });
   (
     await exportCastVoteRecordsToUsbDrive(

apps/scan/backend/src/store.test.ts

@@ -720,6 +720,16 @@ test('getBallotsCounted', () => {
   expect(store.getBallotsCounted()).toEqual(1);
 });
 
+test('isContinuousExportOperationInProgress and setIsContinuousExportOperationInProgress', () => {
+  const store = Store.memoryStore();
+
+  expect(store.isContinuousExportOperationInProgress()).toEqual(false);
+  store.setIsContinuousExportOperationInProgress(true);
+  expect(store.isContinuousExportOperationInProgress()).toEqual(true);
+  store.setIsContinuousExportOperationInProgress(false);
+  expect(store.isContinuousExportOperationInProgress()).toEqual(false);
+});
+
 test('getExportDirectoryName and setExportDirectoryName', () => {
   const store = Store.memoryStore();
 

apps/scan/backend/src/store.ts

@@ -129,6 +129,10 @@ export class Store {
     private readonly uiStringsStore: UiStringsStore
   ) {}
 
+  // Used by shared CVR export logic in libs/backend
+  // eslint-disable-next-line vx/gts-no-public-class-fields
+  readonly scannerType = 'precinct';
+
   getDbPath(): string {
     return this.client.getDatabasePath();
   }
@@ -823,15 +827,44 @@ export class Store {
     return safeParseSystemSettings(result.data).unsafeUnwrap();
   }
 
+  isContinuousExportOperationInProgress(): boolean {
+    const row = this.client.one(
+      `
+      select
+        is_continuous_export_operation_in_progress as isContinuousExportOperationInProgress
+      from is_continuous_export_operation_in_progress
+      `
+    ) as { isContinuousExportOperationInProgress: number } | undefined;
+    return Boolean(row?.isContinuousExportOperationInProgress);
+  }
+
+  setIsContinuousExportOperationInProgress(
+    isContinuousExportOperationInProgress: boolean
+  ): void {
+    this.client.run('delete from is_continuous_export_operation_in_progress');
+    this.client.run(
+      `
+      insert into is_continuous_export_operation_in_progress (
+        is_continuous_export_operation_in_progress
+      ) values (?)
+      `,
+      isContinuousExportOperationInProgress ? 1 : 0
+    );
+  }
+
   /**
    * Gets the name of the directory that we're continuously exporting to, e.g.
    * TEST__machine_SCAN-0001__2023-08-16_17-02-24. Returns undefined if not yet set.
    */
   getExportDirectoryName(): string | undefined {
-    const result = this.client.one(
-      'select export_directory_name as exportDirectoryName from export_directory_name'
+    const row = this.client.one(
+      `
+      select
+        export_directory_name as exportDirectoryName
+      from export_directory_name
+      `
     ) as { exportDirectoryName: string } | undefined;
-    return result?.exportDirectoryName;
+    return row?.exportDirectoryName;
   }
 
   /**
@@ -840,7 +873,11 @@ export class Store {
   setExportDirectoryName(exportDirectoryName: string): void {
     this.client.run('delete from export_directory_name');
     this.client.run(
-      'insert into export_directory_name (export_directory_name) values (?)',
+      `
+      insert into export_directory_name (
+        export_directory_name
+      ) values (?)
+      `,
       exportDirectoryName
     );
   }

apps/scan/backend/test/helpers/custom_helpers.ts

@@ -39,6 +39,7 @@ import {
   waitForContinuousExportToUsbDrive,
   waitForStatus,
 } from './shared_helpers';
+import { Store } from '../../src/store';
 
 export async function withApp(
   {
@@ -123,7 +124,7 @@ export async function withApp(
     });
     mockUsbDrive.assertComplete();
   } finally {
-    await waitForContinuousExportToUsbDrive(mockUsbDrive);
+    await waitForContinuousExportToUsbDrive(workspace.store);
     await new Promise<void>((resolve, reject) => {
       server.close((error) => (error ? reject(error) : resolve()));
     });
@@ -216,8 +217,8 @@ export function simulateScan(
 
 export async function scanBallot(
   mockScanner: jest.Mocked<CustomScanner>,
-  mockUsbDrive: MockUsbDrive,
   apiClient: grout.Client<Api>,
+  store: Store,
   initialBallotsCounted: number
 ): Promise<void> {
   mockScanner.getStatus.mockResolvedValue(ok(mocks.MOCK_READY_TO_SCAN));
@@ -242,5 +243,5 @@ export async function scanBallot(
     ballotsCounted: initialBallotsCounted + 1,
   });
 
-  await waitForContinuousExportToUsbDrive(mockUsbDrive);
+  await waitForContinuousExportToUsbDrive(store);
 }

apps/scan/backend/test/helpers/shared_helpers.ts

@@ -1,9 +1,6 @@
 import { InsertedSmartCardAuthApi } from '@votingworks/auth';
 import { ok } from '@votingworks/basics';
-import {
-  areOrWereCastVoteRecordsBeingExportedToUsbDrive,
-  mockBallotPackageFileTree,
-} from '@votingworks/backend';
+import { mockBallotPackageFileTree } from '@votingworks/backend';
 import { electionFamousNames2021Fixtures } from '@votingworks/fixtures';
 import * as grout from '@votingworks/grout';
 import {
@@ -24,6 +21,7 @@ import waitForExpect from 'wait-for-expect';
 import { MockUsbDrive } from '@votingworks/usb-drive';
 import { Api } from '../../src/app';
 import { PrecinctScannerStatus } from '../../src/types';
+import { Store } from '../../src/store';
 
 export async function expectStatus(
   apiClient: grout.Client<Api>,
@@ -99,18 +97,11 @@ export async function configureApp(
  * while they're still being read from / written to.
  */
 export async function waitForContinuousExportToUsbDrive(
-  mockUsbDrive: MockUsbDrive
+  store: Store
 ): Promise<void> {
-  // Check that mockUsbDrive.usbDrive.status has been configured before calling it
-  if (mockUsbDrive.usbDrive.status.hasExpectedCalls()) {
-    const usbDriveStatus = await mockUsbDrive.usbDrive.status();
-    await waitForExpect(
-      () =>
-        expect(
-          areOrWereCastVoteRecordsBeingExportedToUsbDrive(usbDriveStatus)
-        ).toEqual(false),
-      10000,
-      250
-    );
-  }
+  await waitForExpect(
+    () => expect(store.isContinuousExportOperationInProgress()).toEqual(false),
+    10000,
+    250
+  );
 }

libs/backend/src/cast_vote_records/export.test.ts

@@ -31,12 +31,10 @@ import {
 } from '../../test/utils';
 import {
   AcceptedSheet,
-  areOrWereCastVoteRecordsBeingExportedToUsbDrive,
   clearDoesUsbDriveRequireCastVoteRecordSyncCachedResult,
   doesUsbDriveRequireCastVoteRecordSync,
   exportCastVoteRecordsToUsbDrive,
   ExportOptions,
-  markCastVoteRecordExportAsInProgress,
   RejectedSheet,
   Sheet,
 } from './export';
@@ -794,7 +792,7 @@ test.each<{
       mockPrecinctScannerStore.setBallotsCounted(1);
       const usbDriveStatus = await mockUsbDrive.usbDrive.status();
       assert(usbDriveStatus.status === 'mounted');
-      await markCastVoteRecordExportAsInProgress(usbDriveStatus.mountPoint);
+      mockPrecinctScannerStore.setIsContinuousExportOperationInProgress(true);
     },
     shouldUsbDriveRequireCastVoteRecordSync: true,
   },
@@ -1017,12 +1015,6 @@ test('cast vote record export clears doesUsbDriveRequireCastVoteRecordSync cache
   ).toEqual(false);
 });
 
-test('areOrWereCastVoteRecordsBeingExportedToUsbDrive returns false when no USB drive', () => {
-  expect(
-    areOrWereCastVoteRecordsBeingExportedToUsbDrive({ status: 'no_drive' })
-  ).toEqual(false);
-});
-
 test('export and subsequent import of that export', async () => {
   // Export
   process.env['VX_MACHINE_TYPE'] = 'scan';