Update libs/cvr-fixture-generator to use new CVR export format (plus …

…other small tweaks) (#3988)

* Make various small tweaks to libs/backend functions:

- Remove new CVR functions dependence on legacy CVR functions by
  moving relevant helpers
- Have SKIP_*_AUTHENTICATION feature flags actually skip auth
  check, instead of still performing the auth check and ignoring
  the result
- On CVR import, properly account for the fact that layout files
  won't exist for BMD ballots
- Fix a buggy error check that was referencing the wrong variable
- Rename castVoteRecords castVoteRecordIterator for clarity

* Updave VxAdmin backend to account for libs/backend tweaks

* Add libs/auth as a libs/cvr-fixture-generator dependency

* Update libs/cvr-fixture-generator to use the new CVR export format

* Update libs/cvr-fixture-generator tests

* Have libs/cvr-fixture-generator use UUIDs

apps/admin/backend/src/cast_vote_records.ts

@@ -95,7 +95,8 @@ export async function importCastVoteRecords(
   if (readResult.isErr()) {
     return readResult;
   }
-  const { castVoteRecordExportMetadata, castVoteRecords } = readResult.ok();
+  const { castVoteRecordExportMetadata, castVoteRecordIterator } =
+    readResult.ok();
   const { castVoteRecordReportMetadata } = castVoteRecordExportMetadata;
 
   const exportDirectoryName = path.basename(exportDirectoryPath);
@@ -158,7 +159,7 @@ export async function importCastVoteRecords(
     let newlyAdded = 0;
     let alreadyPresent = 0;
     const precinctIds = new Set<string>();
-    for await (const castVoteRecordResult of castVoteRecords) {
+    for await (const castVoteRecordResult of castVoteRecordIterator) {
       if (castVoteRecordResult.isErr()) {
         return err({
           ...castVoteRecordResult.err(),
@@ -216,6 +217,7 @@ export async function importCastVoteRecords(
         if (hmpbCastVoteRecordWriteIns.length > 0) {
           // Guaranteed to exist given validation in readCastVoteRecordExport
           assert(referencedFiles !== undefined);
+          assert(referencedFiles.layoutFilePaths !== undefined);
 
           for (const i of [0, 1] as const) {
             const imageData = await fs.readFile(

libs/backend/src/ballot_package/ballot_package_io.ts

@@ -94,17 +94,15 @@ export async function readBallotPackageFromUsb(
     return filepathResult;
   }
 
-  const artifactAuthenticationResult =
-    await authenticateArtifactUsingSignatureFile({
-      type: 'election_package',
-      filePath: filepathResult.ok(),
-    });
-  if (
-    artifactAuthenticationResult.isErr() &&
-    !isFeatureFlagEnabled(
-      BooleanEnvironmentVariableName.SKIP_BALLOT_PACKAGE_AUTHENTICATION
-    )
-  ) {
+  const artifactAuthenticationResult = isFeatureFlagEnabled(
+    BooleanEnvironmentVariableName.SKIP_BALLOT_PACKAGE_AUTHENTICATION
+  )
+    ? ok()
+    : await authenticateArtifactUsingSignatureFile({
+        type: 'election_package',
+        filePath: filepathResult.ok(),
+      });
+  if (artifactAuthenticationResult.isErr()) {
     await logger.log(LogEventId.BallotPackageLoadedFromUsb, 'system', {
       disposition: 'failure',
       message: 'Ballot package authentication erred.',

libs/backend/src/cast_vote_records/export.ts

@@ -24,8 +24,11 @@ import {
   CastVoteRecordExportMetadata,
   CVR,
   ElectionDefinition,
+  Id,
   MarkThresholds,
+  PageInterpretation,
   PollsState,
+  SheetOf,
   unsafeParse,
 } from '@votingworks/types';
 import { UsbDrive, UsbDriveStatus } from '@votingworks/usb-drive';
@@ -48,13 +51,8 @@ import {
   describeSheetValidationError,
 } from './canonicalize';
 import { readCastVoteRecordExportMetadata } from './import';
-import { ResultSheet } from './legacy_export';
 import { buildElectionOptionPositionMap } from './option_map';
 
-type ExportCastVoteRecordsToUsbDriveError =
-  | { type: 'invalid-sheet-found'; message: string }
-  | ExportDataError;
-
 /**
  * The subset of scanner store methods relevant to exporting cast vote records
  */
@@ -110,6 +108,32 @@ interface ExportContext {
   usbMountPoint: string;
 }
 
+/**
+ * A sheet to be included in a CVR export
+ */
+export interface ResultSheet {
+  readonly id: Id;
+  readonly batchId: Id;
+  readonly interpretation: SheetOf<PageInterpretation>;
+  readonly frontImagePath: string;
+  readonly backImagePath: string;
+
+  /**
+   * Required per VVSG 2.0 1.1.5-G.7 but only relevant for central scanners. On precinct scanners,
+   * this would compromise voter privacy.
+   */
+  readonly indexInBatch?: number;
+
+  /**
+   * TODO: Determine whether this field is still used and, if not, remove
+   */
+  readonly batchLabel?: string;
+}
+
+type ExportCastVoteRecordsToUsbDriveError =
+  | { type: 'invalid-sheet-found'; message: string }
+  | ExportDataError;
+
 //
 // Helpers
 //

libs/backend/src/cast_vote_records/import.test.ts

@@ -0,0 +1,41 @@
+import { CVR } from '@votingworks/types';
+
+import { TEST_OTHER_REPORT_TYPE } from './build_report_metadata';
+import { isTestReport } from './import';
+
+test.each<{ report: CVR.CastVoteRecordReport; expectedResult: boolean }>([
+  {
+    report: {
+      '@type': 'CVR.CastVoteRecordReport',
+      Election: [],
+      GeneratedDate: new Date().toISOString(),
+      GpUnit: [],
+      OtherReportType: TEST_OTHER_REPORT_TYPE,
+      ReportGeneratingDeviceIds: [],
+      ReportingDevice: [],
+      ReportType: [
+        CVR.ReportType.OriginatingDeviceExport,
+        CVR.ReportType.Other,
+      ],
+      Version: CVR.CastVoteRecordVersion.v1_0_0,
+      vxBatch: [],
+    },
+    expectedResult: true,
+  },
+  {
+    report: {
+      '@type': 'CVR.CastVoteRecordReport',
+      Election: [],
+      GeneratedDate: new Date().toISOString(),
+      GpUnit: [],
+      ReportGeneratingDeviceIds: [],
+      ReportingDevice: [],
+      ReportType: [CVR.ReportType.OriginatingDeviceExport],
+      Version: CVR.CastVoteRecordVersion.v1_0_0,
+      vxBatch: [],
+    },
+    expectedResult: false,
+  },
+])('isTestReport', ({ report, expectedResult }) => {
+  expect(isTestReport(report)).toEqual(expectedResult);
+});

libs/backend/src/cast_vote_records/import.ts

@@ -27,6 +27,8 @@ import {
   isFeatureFlagEnabled,
 } from '@votingworks/utils';
 
+import { TEST_OTHER_REPORT_TYPE } from './build_report_metadata';
+
 type ReadCastVoteRecordExportMetadataError =
   | { type: 'metadata-file-not-found' }
   | { type: 'metadata-file-parse-error' };
@@ -55,7 +57,7 @@ export type ReadCastVoteRecordExportError =
 
 interface ReferencedFiles {
   imageFilePaths: [string, string]; // [front, back]
-  layoutFilePaths: [string, string]; // [front, back]
+  layoutFilePaths?: [string, string]; // [front, back]
 }
 
 interface CastVoteRecordAndReferencedFiles {
@@ -68,7 +70,7 @@ interface CastVoteRecordAndReferencedFiles {
 
 interface CastVoteRecordExportContents {
   castVoteRecordExportMetadata: CastVoteRecordExportMetadata;
-  castVoteRecords: AsyncIteratorPlus<
+  castVoteRecordIterator: AsyncIteratorPlus<
     Result<CastVoteRecordAndReferencedFiles, ReadCastVoteRecordError>
   >;
 }
@@ -146,11 +148,12 @@ async function* castVoteRecordGenerator(
 
     // Only relevant for HMPBs
     let castVoteRecordBallotSheetId: number | undefined;
+    const isHandMarkedPaperBallot = Boolean(castVoteRecord.BallotSheetId);
     if (castVoteRecord.BallotSheetId) {
       const parseBallotSheetIdResult = safeParseNumber(
         castVoteRecord.BallotSheetId
       );
-      if (parseResult.isErr()) {
+      if (parseBallotSheetIdResult.isErr()) {
         yield wrapError({ subType: 'invalid-ballot-sheet-id' });
         return;
       }
@@ -188,18 +191,23 @@ async function* castVoteRecordGenerator(
       const imageFilePaths = ballotImageLocations.map((location) =>
         path.join(castVoteRecordDirectoryPath, location)
       ) as [string, string];
-      const layoutFilePaths = ballotImageLocations.map((location) =>
-        path.join(
-          castVoteRecordDirectoryPath,
-          `${path.parse(location).name}.layout.json`
-        )
-      ) as [string, string];
+      const layoutFilePaths = isHandMarkedPaperBallot
+        ? (ballotImageLocations.map((location) =>
+            path.join(
+              castVoteRecordDirectoryPath,
+              `${path.parse(location).name}.layout.json`
+            )
+          ) as [string, string])
+        : undefined;
 
       if (!imageFilePaths.every((filePath) => existsSync(filePath))) {
         yield wrapError({ subType: 'image-file-not-found' });
         return;
       }
-      if (!layoutFilePaths.every((filePath) => existsSync(filePath))) {
+      if (
+        layoutFilePaths &&
+        !layoutFilePaths.every((filePath) => existsSync(filePath))
+      ) {
         yield wrapError({ subType: 'layout-file-not-found' });
         return;
       }
@@ -230,17 +238,16 @@ export async function readCastVoteRecordExport(
 ): Promise<
   Result<CastVoteRecordExportContents, ReadCastVoteRecordExportError>
 > {
-  const authenticationResult = await authenticateArtifactUsingSignatureFile({
-    type: 'cast_vote_records',
-    context: 'import',
-    directoryPath: exportDirectoryPath,
-  });
-  if (
-    authenticationResult.isErr() &&
-    !isFeatureFlagEnabled(
-      BooleanEnvironmentVariableName.SKIP_CAST_VOTE_RECORDS_AUTHENTICATION
-    )
-  ) {
+  const authenticationResult = isFeatureFlagEnabled(
+    BooleanEnvironmentVariableName.SKIP_CAST_VOTE_RECORDS_AUTHENTICATION
+  )
+    ? ok()
+    : await authenticateArtifactUsingSignatureFile({
+        type: 'cast_vote_records',
+        context: 'import',
+        directoryPath: exportDirectoryPath,
+      });
+  if (authenticationResult.isErr()) {
     return err({ type: 'authentication-error' });
   }
 
@@ -256,12 +263,31 @@ export async function readCastVoteRecordExport(
       (batch) => batch['@id']
     )
   );
-  const castVoteRecords = iter(
+  const castVoteRecordIterator = iter(
     castVoteRecordGenerator(exportDirectoryPath, batchIds)
   );
 
   return ok({
     castVoteRecordExportMetadata,
-    castVoteRecords,
+    castVoteRecordIterator,
   });
 }
+
+/**
+ * Determines whether or not a cast vote record report was generated on a machine in test mode
+ */
+export function isTestReport(
+  castVoteRecordReport: Omit<CVR.CastVoteRecordReport, 'CVR'>
+): boolean {
+  const containsOtherReportType = Boolean(
+    castVoteRecordReport.ReportType?.some(
+      (reportType) => reportType === CVR.ReportType.Other
+    )
+  );
+  const otherReportTypeContainsTest = Boolean(
+    castVoteRecordReport.OtherReportType?.split(',').includes(
+      TEST_OTHER_REPORT_TYPE
+    )
+  );
+  return containsOtherReportType && otherReportTypeContainsTest;
+}

libs/backend/src/cast_vote_records/legacy_export.test.ts

@@ -28,8 +28,8 @@ import {
   exportCastVoteRecordReportToUsbDrive,
   getCastVoteRecordReportStream,
   InvalidSheetFoundError,
-  ResultSheet,
 } from './legacy_export';
+import { ResultSheet } from './export';
 
 jest.mock('@votingworks/auth', (): typeof import('@votingworks/auth') => ({
   ...jest.requireActual('@votingworks/auth'),

libs/backend/src/cast_vote_records/legacy_export.ts

@@ -4,9 +4,6 @@ import {
   BatchInfo,
   CVR,
   ElectionDefinition,
-  Id,
-  PageInterpretation,
-  SheetOf,
   unsafeParse,
 } from '@votingworks/types';
 import { err, ok, Optional, Result } from '@votingworks/basics';
@@ -37,26 +34,7 @@ import {
 import { SCAN_ALLOWED_EXPORT_PATTERNS, VX_MACHINE_ID } from '../scan_globals';
 import { buildCastVoteRecordReportMetadata } from './build_report_metadata';
 import { buildElectionOptionPositionMap } from './option_map';
-
-/**
- * Properties of each sheet that are needed to generate a cast vote record
- * for that sheet.
- */
-export interface ResultSheet {
-  readonly id: Id;
-  readonly batchId: Id;
-  /**
-   * `indexInBatch` only applies to the central scanner. It is required in cast
-   * vote records per VVSG 2.0 1.1.5-G.7, but is not included for the precinct
-   * scanner because that would compromise voter privacy.
-   */
-  readonly indexInBatch?: number;
-  // TODO: remove once the deprecated CVR export is no longer using batchLabel
-  readonly batchLabel?: string;
-  readonly interpretation: SheetOf<PageInterpretation>;
-  readonly frontImagePath: string;
-  readonly backImagePath: string;
-}
+import { ResultSheet } from './export';
 
 /**
  * In cast vote record exports, the subdirectory under which images are

libs/backend/src/cast_vote_records/legacy_import.test.ts

@@ -6,14 +6,12 @@ import { rmSync, writeFileSync } from 'fs';
 import { join } from 'path';
 import {
   getCastVoteRecordReportImport,
-  isTestReport,
   validateCastVoteRecordReportDirectoryStructure,
 } from './legacy_import';
 import {
   CVR_BALLOT_IMAGES_SUBDIRECTORY,
   CVR_BALLOT_LAYOUTS_SUBDIRECTORY,
 } from './legacy_export';
-import { TEST_OTHER_REPORT_TYPE } from './build_report_metadata';
 
 const cdfCvrReport =
   electionGridLayoutNewHampshireAmherstFixtures.castVoteRecordReport;
@@ -162,41 +160,3 @@ describe('validateCastVoteRecordReportDirectoryStructure', () => {
     expect(validationResult.err()).toMatchObject({ type: 'invalid-directory' });
   });
 });
-
-describe('isTestReport', () => {
-  test('when test', () => {
-    expect(
-      isTestReport({
-        '@type': 'CVR.CastVoteRecordReport',
-        ReportType: [
-          CVR.ReportType.OriginatingDeviceExport,
-          CVR.ReportType.Other,
-        ],
-        OtherReportType: TEST_OTHER_REPORT_TYPE,
-        GeneratedDate: Date.now().toString(),
-        GpUnit: [],
-        Election: [],
-        ReportGeneratingDeviceIds: [],
-        ReportingDevice: [],
-        Version: CVR.CastVoteRecordVersion.v1_0_0,
-        vxBatch: [],
-      })
-    ).toBeTruthy();
-  });
-
-  test('when not test', () => {
-    expect(
-      isTestReport({
-        '@type': 'CVR.CastVoteRecordReport',
-        ReportType: [CVR.ReportType.OriginatingDeviceExport],
-        GeneratedDate: Date.now().toString(),
-        GpUnit: [],
-        Election: [],
-        ReportGeneratingDeviceIds: [],
-        ReportingDevice: [],
-        Version: CVR.CastVoteRecordVersion.v1_0_0,
-        vxBatch: [],
-      })
-    ).toBeFalsy();
-  });
-});