diff --git a/apps/admin/backend/src/cast_vote_records.ts b/apps/admin/backend/src/cast_vote_records.ts index acf47bd8f8..875e77eac8 100644 --- a/apps/admin/backend/src/cast_vote_records.ts +++ b/apps/admin/backend/src/cast_vote_records.ts @@ -95,7 +95,8 @@ export async function importCastVoteRecords( if (readResult.isErr()) { return readResult; } - const { castVoteRecordExportMetadata, castVoteRecords } = readResult.ok(); + const { castVoteRecordExportMetadata, castVoteRecordIterator } = + readResult.ok(); const { castVoteRecordReportMetadata } = castVoteRecordExportMetadata; const exportDirectoryName = path.basename(exportDirectoryPath); @@ -158,7 +159,7 @@ export async function importCastVoteRecords( let newlyAdded = 0; let alreadyPresent = 0; const precinctIds = new Set(); - for await (const castVoteRecordResult of castVoteRecords) { + for await (const castVoteRecordResult of castVoteRecordIterator) { if (castVoteRecordResult.isErr()) { return err({ ...castVoteRecordResult.err(), @@ -216,6 +217,7 @@ export async function importCastVoteRecords( if (hmpbCastVoteRecordWriteIns.length > 0) { // Guaranteed to exist given validation in readCastVoteRecordExport assert(referencedFiles !== undefined); + assert(referencedFiles.layoutFilePaths !== undefined); for (const i of [0, 1] as const) { const imageData = await fs.readFile( diff --git a/libs/backend/src/ballot_package/ballot_package_io.ts b/libs/backend/src/ballot_package/ballot_package_io.ts index 0b894a7743..34fa9a5bc2 100644 --- a/libs/backend/src/ballot_package/ballot_package_io.ts +++ b/libs/backend/src/ballot_package/ballot_package_io.ts @@ -94,17 +94,15 @@ export async function readBallotPackageFromUsb( return filepathResult; } - const artifactAuthenticationResult = - await authenticateArtifactUsingSignatureFile({ - type: 'election_package', - filePath: filepathResult.ok(), - }); - if ( - artifactAuthenticationResult.isErr() && - !isFeatureFlagEnabled( - BooleanEnvironmentVariableName.SKIP_BALLOT_PACKAGE_AUTHENTICATION - ) - ) { + const artifactAuthenticationResult = isFeatureFlagEnabled( + BooleanEnvironmentVariableName.SKIP_BALLOT_PACKAGE_AUTHENTICATION + ) + ? ok() + : await authenticateArtifactUsingSignatureFile({ + type: 'election_package', + filePath: filepathResult.ok(), + }); + if (artifactAuthenticationResult.isErr()) { await logger.log(LogEventId.BallotPackageLoadedFromUsb, 'system', { disposition: 'failure', message: 'Ballot package authentication erred.', diff --git a/libs/backend/src/cast_vote_records/export.ts b/libs/backend/src/cast_vote_records/export.ts index 6705d19625..77ca72a2e9 100644 --- a/libs/backend/src/cast_vote_records/export.ts +++ b/libs/backend/src/cast_vote_records/export.ts @@ -24,8 +24,11 @@ import { CastVoteRecordExportMetadata, CVR, ElectionDefinition, + Id, MarkThresholds, + PageInterpretation, PollsState, + SheetOf, unsafeParse, } from '@votingworks/types'; import { UsbDrive, UsbDriveStatus } from '@votingworks/usb-drive'; @@ -48,13 +51,8 @@ import { describeSheetValidationError, } from './canonicalize'; import { readCastVoteRecordExportMetadata } from './import'; -import { ResultSheet } from './legacy_export'; import { buildElectionOptionPositionMap } from './option_map'; -type ExportCastVoteRecordsToUsbDriveError = - | { type: 'invalid-sheet-found'; message: string } - | ExportDataError; - /** * The subset of scanner store methods relevant to exporting cast vote records */ @@ -110,6 +108,32 @@ interface ExportContext { usbMountPoint: string; } +/** + * A sheet to be included in a CVR export + */ +export interface ResultSheet { + readonly id: Id; + readonly batchId: Id; + readonly interpretation: SheetOf; + readonly frontImagePath: string; + readonly backImagePath: string; + + /** + * Required per VVSG 2.0 1.1.5-G.7 but only relevant for central scanners. On precinct scanners, + * this would compromise voter privacy. + */ + readonly indexInBatch?: number; + + /** + * TODO: Determine whether this field is still used and, if not, remove + */ + readonly batchLabel?: string; +} + +type ExportCastVoteRecordsToUsbDriveError = + | { type: 'invalid-sheet-found'; message: string } + | ExportDataError; + // // Helpers // diff --git a/libs/backend/src/cast_vote_records/import.test.ts b/libs/backend/src/cast_vote_records/import.test.ts new file mode 100644 index 0000000000..ed562cf6f8 --- /dev/null +++ b/libs/backend/src/cast_vote_records/import.test.ts @@ -0,0 +1,41 @@ +import { CVR } from '@votingworks/types'; + +import { TEST_OTHER_REPORT_TYPE } from './build_report_metadata'; +import { isTestReport } from './import'; + +test.each<{ report: CVR.CastVoteRecordReport; expectedResult: boolean }>([ + { + report: { + '@type': 'CVR.CastVoteRecordReport', + Election: [], + GeneratedDate: new Date().toISOString(), + GpUnit: [], + OtherReportType: TEST_OTHER_REPORT_TYPE, + ReportGeneratingDeviceIds: [], + ReportingDevice: [], + ReportType: [ + CVR.ReportType.OriginatingDeviceExport, + CVR.ReportType.Other, + ], + Version: CVR.CastVoteRecordVersion.v1_0_0, + vxBatch: [], + }, + expectedResult: true, + }, + { + report: { + '@type': 'CVR.CastVoteRecordReport', + Election: [], + GeneratedDate: new Date().toISOString(), + GpUnit: [], + ReportGeneratingDeviceIds: [], + ReportingDevice: [], + ReportType: [CVR.ReportType.OriginatingDeviceExport], + Version: CVR.CastVoteRecordVersion.v1_0_0, + vxBatch: [], + }, + expectedResult: false, + }, +])('isTestReport', ({ report, expectedResult }) => { + expect(isTestReport(report)).toEqual(expectedResult); +}); diff --git a/libs/backend/src/cast_vote_records/import.ts b/libs/backend/src/cast_vote_records/import.ts index 8744e48f4b..8465a870b2 100644 --- a/libs/backend/src/cast_vote_records/import.ts +++ b/libs/backend/src/cast_vote_records/import.ts @@ -27,6 +27,8 @@ import { isFeatureFlagEnabled, } from '@votingworks/utils'; +import { TEST_OTHER_REPORT_TYPE } from './build_report_metadata'; + type ReadCastVoteRecordExportMetadataError = | { type: 'metadata-file-not-found' } | { type: 'metadata-file-parse-error' }; @@ -55,7 +57,7 @@ export type ReadCastVoteRecordExportError = interface ReferencedFiles { imageFilePaths: [string, string]; // [front, back] - layoutFilePaths: [string, string]; // [front, back] + layoutFilePaths?: [string, string]; // [front, back] } interface CastVoteRecordAndReferencedFiles { @@ -68,7 +70,7 @@ interface CastVoteRecordAndReferencedFiles { interface CastVoteRecordExportContents { castVoteRecordExportMetadata: CastVoteRecordExportMetadata; - castVoteRecords: AsyncIteratorPlus< + castVoteRecordIterator: AsyncIteratorPlus< Result >; } @@ -146,11 +148,12 @@ async function* castVoteRecordGenerator( // Only relevant for HMPBs let castVoteRecordBallotSheetId: number | undefined; + const isHandMarkedPaperBallot = Boolean(castVoteRecord.BallotSheetId); if (castVoteRecord.BallotSheetId) { const parseBallotSheetIdResult = safeParseNumber( castVoteRecord.BallotSheetId ); - if (parseResult.isErr()) { + if (parseBallotSheetIdResult.isErr()) { yield wrapError({ subType: 'invalid-ballot-sheet-id' }); return; } @@ -188,18 +191,23 @@ async function* castVoteRecordGenerator( const imageFilePaths = ballotImageLocations.map((location) => path.join(castVoteRecordDirectoryPath, location) ) as [string, string]; - const layoutFilePaths = ballotImageLocations.map((location) => - path.join( - castVoteRecordDirectoryPath, - `${path.parse(location).name}.layout.json` - ) - ) as [string, string]; + const layoutFilePaths = isHandMarkedPaperBallot + ? (ballotImageLocations.map((location) => + path.join( + castVoteRecordDirectoryPath, + `${path.parse(location).name}.layout.json` + ) + ) as [string, string]) + : undefined; if (!imageFilePaths.every((filePath) => existsSync(filePath))) { yield wrapError({ subType: 'image-file-not-found' }); return; } - if (!layoutFilePaths.every((filePath) => existsSync(filePath))) { + if ( + layoutFilePaths && + !layoutFilePaths.every((filePath) => existsSync(filePath)) + ) { yield wrapError({ subType: 'layout-file-not-found' }); return; } @@ -230,17 +238,16 @@ export async function readCastVoteRecordExport( ): Promise< Result > { - const authenticationResult = await authenticateArtifactUsingSignatureFile({ - type: 'cast_vote_records', - context: 'import', - directoryPath: exportDirectoryPath, - }); - if ( - authenticationResult.isErr() && - !isFeatureFlagEnabled( - BooleanEnvironmentVariableName.SKIP_CAST_VOTE_RECORDS_AUTHENTICATION - ) - ) { + const authenticationResult = isFeatureFlagEnabled( + BooleanEnvironmentVariableName.SKIP_CAST_VOTE_RECORDS_AUTHENTICATION + ) + ? ok() + : await authenticateArtifactUsingSignatureFile({ + type: 'cast_vote_records', + context: 'import', + directoryPath: exportDirectoryPath, + }); + if (authenticationResult.isErr()) { return err({ type: 'authentication-error' }); } @@ -256,12 +263,31 @@ export async function readCastVoteRecordExport( (batch) => batch['@id'] ) ); - const castVoteRecords = iter( + const castVoteRecordIterator = iter( castVoteRecordGenerator(exportDirectoryPath, batchIds) ); return ok({ castVoteRecordExportMetadata, - castVoteRecords, + castVoteRecordIterator, }); } + +/** + * Determines whether or not a cast vote record report was generated on a machine in test mode + */ +export function isTestReport( + castVoteRecordReport: Omit +): boolean { + const containsOtherReportType = Boolean( + castVoteRecordReport.ReportType?.some( + (reportType) => reportType === CVR.ReportType.Other + ) + ); + const otherReportTypeContainsTest = Boolean( + castVoteRecordReport.OtherReportType?.split(',').includes( + TEST_OTHER_REPORT_TYPE + ) + ); + return containsOtherReportType && otherReportTypeContainsTest; +} diff --git a/libs/backend/src/cast_vote_records/legacy_export.test.ts b/libs/backend/src/cast_vote_records/legacy_export.test.ts index 208c9fcffc..daff279922 100644 --- a/libs/backend/src/cast_vote_records/legacy_export.test.ts +++ b/libs/backend/src/cast_vote_records/legacy_export.test.ts @@ -28,8 +28,8 @@ import { exportCastVoteRecordReportToUsbDrive, getCastVoteRecordReportStream, InvalidSheetFoundError, - ResultSheet, } from './legacy_export'; +import { ResultSheet } from './export'; jest.mock('@votingworks/auth', (): typeof import('@votingworks/auth') => ({ ...jest.requireActual('@votingworks/auth'), diff --git a/libs/backend/src/cast_vote_records/legacy_export.ts b/libs/backend/src/cast_vote_records/legacy_export.ts index 48b1b7e1c7..de924871d4 100644 --- a/libs/backend/src/cast_vote_records/legacy_export.ts +++ b/libs/backend/src/cast_vote_records/legacy_export.ts @@ -4,9 +4,6 @@ import { BatchInfo, CVR, ElectionDefinition, - Id, - PageInterpretation, - SheetOf, unsafeParse, } from '@votingworks/types'; import { err, ok, Optional, Result } from '@votingworks/basics'; @@ -37,26 +34,7 @@ import { import { SCAN_ALLOWED_EXPORT_PATTERNS, VX_MACHINE_ID } from '../scan_globals'; import { buildCastVoteRecordReportMetadata } from './build_report_metadata'; import { buildElectionOptionPositionMap } from './option_map'; - -/** - * Properties of each sheet that are needed to generate a cast vote record - * for that sheet. - */ -export interface ResultSheet { - readonly id: Id; - readonly batchId: Id; - /** - * `indexInBatch` only applies to the central scanner. It is required in cast - * vote records per VVSG 2.0 1.1.5-G.7, but is not included for the precinct - * scanner because that would compromise voter privacy. - */ - readonly indexInBatch?: number; - // TODO: remove once the deprecated CVR export is no longer using batchLabel - readonly batchLabel?: string; - readonly interpretation: SheetOf; - readonly frontImagePath: string; - readonly backImagePath: string; -} +import { ResultSheet } from './export'; /** * In cast vote record exports, the subdirectory under which images are diff --git a/libs/backend/src/cast_vote_records/legacy_import.test.ts b/libs/backend/src/cast_vote_records/legacy_import.test.ts index 2d80a829fb..70d06f63de 100644 --- a/libs/backend/src/cast_vote_records/legacy_import.test.ts +++ b/libs/backend/src/cast_vote_records/legacy_import.test.ts @@ -6,14 +6,12 @@ import { rmSync, writeFileSync } from 'fs'; import { join } from 'path'; import { getCastVoteRecordReportImport, - isTestReport, validateCastVoteRecordReportDirectoryStructure, } from './legacy_import'; import { CVR_BALLOT_IMAGES_SUBDIRECTORY, CVR_BALLOT_LAYOUTS_SUBDIRECTORY, } from './legacy_export'; -import { TEST_OTHER_REPORT_TYPE } from './build_report_metadata'; const cdfCvrReport = electionGridLayoutNewHampshireAmherstFixtures.castVoteRecordReport; @@ -162,41 +160,3 @@ describe('validateCastVoteRecordReportDirectoryStructure', () => { expect(validationResult.err()).toMatchObject({ type: 'invalid-directory' }); }); }); - -describe('isTestReport', () => { - test('when test', () => { - expect( - isTestReport({ - '@type': 'CVR.CastVoteRecordReport', - ReportType: [ - CVR.ReportType.OriginatingDeviceExport, - CVR.ReportType.Other, - ], - OtherReportType: TEST_OTHER_REPORT_TYPE, - GeneratedDate: Date.now().toString(), - GpUnit: [], - Election: [], - ReportGeneratingDeviceIds: [], - ReportingDevice: [], - Version: CVR.CastVoteRecordVersion.v1_0_0, - vxBatch: [], - }) - ).toBeTruthy(); - }); - - test('when not test', () => { - expect( - isTestReport({ - '@type': 'CVR.CastVoteRecordReport', - ReportType: [CVR.ReportType.OriginatingDeviceExport], - GeneratedDate: Date.now().toString(), - GpUnit: [], - Election: [], - ReportGeneratingDeviceIds: [], - ReportingDevice: [], - Version: CVR.CastVoteRecordVersion.v1_0_0, - vxBatch: [], - }) - ).toBeFalsy(); - }); -}); diff --git a/libs/backend/src/cast_vote_records/legacy_import.ts b/libs/backend/src/cast_vote_records/legacy_import.ts index 020ab90738..5abb672089 100644 --- a/libs/backend/src/cast_vote_records/legacy_import.ts +++ b/libs/backend/src/cast_vote_records/legacy_import.ts @@ -20,10 +20,7 @@ import { CVR_BALLOT_IMAGES_SUBDIRECTORY, CVR_BALLOT_LAYOUTS_SUBDIRECTORY, } from './legacy_export'; -import { - CastVoteRecordReportMetadata, - TEST_OTHER_REPORT_TYPE, -} from './build_report_metadata'; +import { CastVoteRecordReportMetadata } from './build_report_metadata'; /** * Variant of {@link CastVoteRecordReport} in which the `CVR` array is replaced @@ -224,22 +221,3 @@ export async function validateCastVoteRecordReportDirectoryStructure( return ok(relativeImagePaths); } - -/** - * Determines whether a cast vote record report is a test report or not. A report - * is a test report if `CVR.ReportType` contains `ReportType.Other` and - * `CVR.OtherReportType`, as a comma-separated list of strings, contains - * {@link TEST_OTHER_REPORT_TYPE}. - */ -export function isTestReport(metadata: CastVoteRecordReportMetadata): boolean { - const containsOtherReportType = metadata.ReportType?.some( - (reportType) => reportType === CVR.ReportType.Other - ); - if (!containsOtherReportType) return false; - - const otherReportTypeContainsTest = metadata.OtherReportType?.split( - ',' - ).includes(TEST_OTHER_REPORT_TYPE); - - return Boolean(otherReportTypeContainsTest); -} diff --git a/libs/cvr-fixture-generator/package.json b/libs/cvr-fixture-generator/package.json index d6c984be4e..7b29719a5c 100644 --- a/libs/cvr-fixture-generator/package.json +++ b/libs/cvr-fixture-generator/package.json @@ -22,6 +22,7 @@ "pre-commit": "lint-staged" }, "dependencies": { + "@votingworks/auth": "*", "@votingworks/backend": "*", "@votingworks/basics": "*", "@votingworks/converter-nh-accuvote": "workspace:^", @@ -33,6 +34,7 @@ "esbuild-runner": "^2.2.1", "js-sha256": "^0.9.0", "lodash.clonedeep": "^4.5.0", + "uuid": "^9.0.1", "yargs": "^17.5.1" }, "devDependencies": { @@ -40,6 +42,7 @@ "@types/lodash.clonedeep": "^4.5.7", "@types/node": "16.18.23", "@types/tmp": "^0.2.3", + "@types/uuid": "^9.0.4", "@types/yargs": "^17.0.12", "@votingworks/test-utils": "*", "eslint-plugin-vx": "*", diff --git a/libs/cvr-fixture-generator/src/cli/generate/main.test.ts b/libs/cvr-fixture-generator/src/cli/generate/main.test.ts index 120567974b..afd7483808 100644 --- a/libs/cvr-fixture-generator/src/cli/generate/main.test.ts +++ b/libs/cvr-fixture-generator/src/cli/generate/main.test.ts @@ -3,36 +3,24 @@ import { electionGridLayoutNewHampshireAmherstFixtures, } from '@votingworks/fixtures'; import { fakeReadable, fakeWritable } from '@votingworks/test-utils'; -import { safeParseJson, CVR, unsafeParse } from '@votingworks/types'; -import { readFileSync } from 'fs'; +import { CVR } from '@votingworks/types'; import fs from 'fs/promises'; import { join, resolve } from 'path'; import { dirSync } from 'tmp'; import { - CAST_VOTE_RECORD_REPORT_FILENAME, getWriteInsFromCastVoteRecord, isBmdWriteIn, } from '@votingworks/utils'; -import { getCastVoteRecordReportImport } from '@votingworks/backend'; +import { readCastVoteRecordExport } from '@votingworks/backend'; import { assert } from '@votingworks/basics'; -import { DEFAULT_SCANNER_ID, main } from './main'; -import { getBatchIdForScannerId } from '../../utils'; +import { main } from './main'; +import { IMAGE_URI_REGEX } from '../../utils'; jest.setTimeout(30_000); const electionDefinitionPathAmherst = electionGridLayoutNewHampshireAmherstFixtures.electionJson.asFilePath(); -function reportFromFile(directory: string) { - const filename = join(directory, CAST_VOTE_RECORD_REPORT_FILENAME); - const reportParseResult = safeParseJson( - readFileSync(filename, 'utf8'), - CVR.CastVoteRecordReportSchema - ); - expect(reportParseResult.isOk()).toBeTruthy(); - return reportParseResult.unsafeUnwrap(); -} - async function run( args: string[] ): Promise<{ exitCode: number; stdout: string; stderr: string }> { @@ -52,6 +40,29 @@ async function run( }; } +async function readAndValidateCastVoteRecordExport( + exportDirectoryPath: string +): Promise<{ + castVoteRecordReportMetadata: CVR.CastVoteRecordReport; + castVoteRecords: CVR.CVR[]; +}> { + const readResult = await readCastVoteRecordExport(exportDirectoryPath); + assert(readResult.isOk()); + const { castVoteRecordExportMetadata, castVoteRecordIterator } = + readResult.ok(); + const castVoteRecords: CVR.CVR[] = []; + for await (const castVoteRecordResult of castVoteRecordIterator) { + assert(castVoteRecordResult.isOk()); + const { castVoteRecord } = castVoteRecordResult.ok(); + castVoteRecords.push(castVoteRecord); + } + return { + castVoteRecordReportMetadata: + castVoteRecordExportMetadata.castVoteRecordReportMetadata, + castVoteRecords, + }; +} + test('--help', async () => { expect(await run(['--help'])).toEqual({ exitCode: 0, @@ -101,8 +112,10 @@ test('generate with defaults', async () => { stderr: '', }); - const report = reportFromFile(outputDirectory.name); - expect(report.CVR).toHaveLength(184); + const { castVoteRecords } = await readAndValidateCastVoteRecordExport( + outputDirectory.name + ); + expect(castVoteRecords).toHaveLength(184); }); test('generate with custom number of records below the suggested number', async () => { @@ -124,8 +137,10 @@ test('generate with custom number of records below the suggested number', async stderr: expect.stringContaining('WARNING:'), }); - const report = reportFromFile(outputDirectory.name); - expect(report.CVR).toHaveLength(100); + const { castVoteRecords } = await readAndValidateCastVoteRecordExport( + outputDirectory.name + ); + expect(castVoteRecords).toHaveLength(100); }); test('generate with custom number of records above the suggested number', async () => { @@ -149,20 +164,13 @@ test('generate with custom number of records above the suggested number', async stderr: '', }); - const castVoteRecordReportImport = ( - await getCastVoteRecordReportImport( - join(outputDirectory.name, CAST_VOTE_RECORD_REPORT_FILENAME) - ) - ).assertOk('generated cast vote record should be valid'); - - let cvrCount = 0; - for await (const unparsedCastVoteRecord of castVoteRecordReportImport.CVR) { - const castVoteRecord = unsafeParse(CVR.CVRSchema, unparsedCastVoteRecord); - expect(castVoteRecord.UniqueId).toEqual(`pre-${cvrCount}`); - cvrCount += 1; + const { castVoteRecords } = await readAndValidateCastVoteRecordExport( + outputDirectory.name + ); + expect(castVoteRecords).toHaveLength(500); + for (const castVoteRecord of castVoteRecords) { + expect(castVoteRecord.UniqueId).toEqual(expect.stringMatching(/pre-(.+)/)); } - - expect(cvrCount).toEqual(500); }); test('generate live mode CVRs', async () => { @@ -179,8 +187,9 @@ test('generate live mode CVRs', async () => { '10', ]); - const report = reportFromFile(outputDirectory.name); - expect(report.OtherReportType).toBeUndefined(); + const { castVoteRecordReportMetadata } = + await readAndValidateCastVoteRecordExport(outputDirectory.name); + expect(castVoteRecordReportMetadata.OtherReportType).toBeUndefined(); }); test('generate test mode CVRs', async () => { @@ -196,8 +205,11 @@ test('generate test mode CVRs', async () => { '10', ]); - const report = reportFromFile(outputDirectory.name); - expect(report.OtherReportType?.split(',')).toContain('test'); + const { castVoteRecordReportMetadata } = + await readAndValidateCastVoteRecordExport(outputDirectory.name); + expect(castVoteRecordReportMetadata.OtherReportType?.split(',')).toContain( + 'test' + ); }); test('specifying scanner ids', async () => { @@ -213,9 +225,11 @@ test('specifying scanner ids', async () => { 'scanner1,scanner2', ]); - const report = reportFromFile(outputDirectory.name); - for (const cvr of report.CVR!) { - expect(cvr.CreatingDeviceId).toMatch(/scanner[12]/); + const { castVoteRecords } = await readAndValidateCastVoteRecordExport( + outputDirectory.name + ); + for (const castVoteRecord of castVoteRecords) { + expect(castVoteRecord.CreatingDeviceId).toMatch(/scanner[12]/); } }); @@ -230,52 +244,31 @@ test('including ballot images', async () => { outputDirectory.name, ]); - const report = reportFromFile(outputDirectory.name); - const imageFileUris = new Set(); - assert(report.CVR); - for (const cvr of report.CVR) { - const ballotImages = cvr.BallotImage; - if (ballotImages) { - if (ballotImages[0]?.Location) { - imageFileUris.add(ballotImages[0]?.Location); - } - if (ballotImages[1]?.Location) { - imageFileUris.add(ballotImages[1]?.Location); - } + const { castVoteRecords } = await readAndValidateCastVoteRecordExport( + outputDirectory.name + ); + for (const castVoteRecord of castVoteRecords) { + if (castVoteRecord.BallotImage) { + expect(castVoteRecord.BallotImage[0]?.Location).toEqual( + expect.stringMatching(IMAGE_URI_REGEX) + ); + expect(castVoteRecord.BallotImage[1]?.Location).toEqual( + expect.stringMatching(IMAGE_URI_REGEX) + ); + const castVoteRecordDirectoryContents = ( + await fs.readdir(join(outputDirectory.name, castVoteRecord.UniqueId)) + ).sort(); + expect(castVoteRecordDirectoryContents).toEqual( + [ + `${castVoteRecord.UniqueId}-back.jpg`, + `${castVoteRecord.UniqueId}-back.layout.json`, + `${castVoteRecord.UniqueId}-front.jpg`, + `${castVoteRecord.UniqueId}-front.layout.json`, + 'cast-vote-record-report.json', + ].sort() + ); } } - - const defaultBatchId = getBatchIdForScannerId(DEFAULT_SCANNER_ID); - - // files referenced from the report - expect(Array.from(imageFileUris)).toMatchObject([ - `file:ballot-images/${defaultBatchId}/card-number-3__town-id-00701-precinct-id-__2.jpg`, - `file:ballot-images/${defaultBatchId}/card-number-3__town-id-00701-precinct-id-__1.jpg`, - ]); - - // images exported - expect( - await fs.readdir( - join(outputDirectory.name, 'ballot-images', defaultBatchId) - ) - ).toMatchInlineSnapshot(` - [ - "card-number-3__town-id-00701-precinct-id-__1.jpg", - "card-number-3__town-id-00701-precinct-id-__2.jpg", - ] - `); - - // layouts exported - expect( - await fs.readdir( - join(outputDirectory.name, 'ballot-layouts', defaultBatchId) - ) - ).toMatchInlineSnapshot(` - [ - "card-number-3__town-id-00701-precinct-id-__1.layout.json", - "card-number-3__town-id-00701-precinct-id-__2.layout.json", - ] - `); }); test('generating as BMD ballots (non-gridlayouts election)', async () => { @@ -296,17 +289,18 @@ test('generating as BMD ballots (non-gridlayouts election)', async () => { stderr: '', }); - const report = reportFromFile(outputDirectory.name); - assert(report.CVR); - for (const [index, cvr] of report.CVR.entries()) { - expect(cvr.BallotImage).toBeUndefined(); - expect(cvr.UniqueId).toEqual(index.toString()); + const { castVoteRecords } = await readAndValidateCastVoteRecordExport( + outputDirectory.name + ); + for (const castVoteRecord of castVoteRecords) { + expect(castVoteRecord.BallotImage).toBeUndefined(); + expect(castVoteRecord.UniqueId).toEqual(expect.stringMatching(/[0-9]+/)); expect( - getWriteInsFromCastVoteRecord(cvr).every((castVoteRecordWriteIn) => - Boolean(castVoteRecordWriteIn.text) + getWriteInsFromCastVoteRecord(castVoteRecord).every( + (castVoteRecordWriteIn) => Boolean(castVoteRecordWriteIn.text) ) ).toEqual(true); - const writeIns = cvr.CVRSnapshot[0]!.CVRContest.flatMap( + const writeIns = castVoteRecord.CVRSnapshot[0]!.CVRContest.flatMap( (contest) => contest.CVRContestSelection ) .flatMap((contestSelection) => contestSelection.SelectionPosition) diff --git a/libs/cvr-fixture-generator/src/cli/generate/main.ts b/libs/cvr-fixture-generator/src/cli/generate/main.ts index 2248b29c96..db54555d7f 100644 --- a/libs/cvr-fixture-generator/src/cli/generate/main.ts +++ b/libs/cvr-fixture-generator/src/cli/generate/main.ts @@ -1,31 +1,25 @@ import { - BallotPaperSize, BallotType, CVR, + CastVoteRecordExportMetadata, safeParseElectionDefinition, } from '@votingworks/types'; -import { assert, assertDefined, find, iter } from '@votingworks/basics'; -import { - buildCastVoteRecordReportMetadata, - CVR_BALLOT_IMAGES_SUBDIRECTORY, - CVR_BALLOT_LAYOUTS_SUBDIRECTORY, -} from '@votingworks/backend'; +import { assert, assertDefined, iter } from '@votingworks/basics'; +import { buildCastVoteRecordReportMetadata } from '@votingworks/backend'; import * as fs from 'fs'; import yargs from 'yargs/yargs'; -import { - CAST_VOTE_RECORD_REPORT_FILENAME, - jsonStream, -} from '@votingworks/utils'; import { writeImageData, createImageData } from '@votingworks/image-utils'; -import { pipeline } from 'stream/promises'; -import { join } from 'path'; +import { basename, join, parse } from 'path'; import cloneDeep from 'lodash.clonedeep'; +import { + computeCastVoteRecordRootHashFromScratch, + prepareSignatureFile, +} from '@votingworks/auth'; import { generateBallotPageLayouts, generateCvrs } from '../../generate_cvrs'; import { - generateBallotAssetPath, replaceUniqueId, - IMAGE_URI_REGEX, getBatchIdForScannerId, + PAGE_HEIGHT_INCHES, } from '../../utils'; /** @@ -58,6 +52,8 @@ interface IO { /** * Command line interface for generating a cast vote record file. + * + * TODO: Make more full use of export functions in libs/backend to avoid duplicating logic. */ export async function main( argv: readonly string[], @@ -216,125 +212,77 @@ export async function main( // make the parent folder if it does not exist fs.mkdirSync(outputPath, { recursive: true }); - const reportStream = jsonStream({ - ...reportMetadata, - CVR: castVoteRecords, - }); - - // write the report - await pipeline( - reportStream, - fs.createWriteStream(join(outputPath, CAST_VOTE_RECORD_REPORT_FILENAME)) - ); - - if (election.gridLayouts) { - // determine the images referenced in the report - const imageUris = new Set(); - for (const castVoteRecord of castVoteRecords) { - const ballotImages = castVoteRecord.BallotImage; - if (ballotImages) { - if (ballotImages[0]?.Location) { - imageUris.add(ballotImages[0]?.Location); - } - if (ballotImages[1]?.Location) { - imageUris.add(ballotImages[1]?.Location); - } - } - } - - // export information from the relevant ballot package entries - for (const imageUri of imageUris) { - const regexMatch = imageUri.match(IMAGE_URI_REGEX); - // istanbul ignore next - if (regexMatch === null) { - throw new Error('unexpected file URI format'); - } - const [, batchId, ballotStyleId, precinctId, pageNumberString] = - regexMatch; - assert(batchId !== undefined); - assert(ballotStyleId !== undefined); - assert(precinctId !== undefined); - assert(pageNumberString !== undefined); - // eslint-disable-next-line vx/gts-safe-number-parse - const pageNumber = Number(pageNumberString); - - const pageDpi = 200; - const pageWidthInches = 8.5; - let pageHeightInches: number; - - switch (election.ballotLayout.paperSize) { - case BallotPaperSize.Legal: - pageHeightInches = 14; - break; - - case BallotPaperSize.Custom17: - pageHeightInches = 17; - break; - - case BallotPaperSize.Letter: - default: - pageHeightInches = 11; - break; + for (const castVoteRecord of castVoteRecords) { + const castVoteRecordDirectory = join(outputPath, castVoteRecord.UniqueId); + fs.mkdirSync(castVoteRecordDirectory); + const castVoteRecordReport: CVR.CastVoteRecordReport = { + ...reportMetadata, + CVR: [castVoteRecord], + }; + fs.writeFileSync( + join(castVoteRecordDirectory, 'cast-vote-record-report.json'), + JSON.stringify(castVoteRecordReport) + ); + if (castVoteRecord.BallotImage) { + const layouts = generateBallotPageLayouts(election, { + ballotStyleId: castVoteRecord.BallotStyleId, + ballotType: BallotType.Precinct, + electionHash, + isTestMode: testMode, + precinctId: castVoteRecord.BallotStyleUnitId, + }); + for (const i of [0, 1] as const) { + const imageFilePath = join( + castVoteRecordDirectory, + assertDefined(castVoteRecord.BallotImage[i]?.Location).replace( + 'file:', + '' + ) + ); + const layoutFilePath = imageFilePath.replace('.jpg', '.layout.json'); + + const pageHeightInches = + PAGE_HEIGHT_INCHES[election.ballotLayout.paperSize]; + const pageWidthInches = 8.5; + const pageDpi = 200; + await writeImageData( + imageFilePath, + createImageData( + new Uint8ClampedArray( + pageWidthInches * pageDpi * pageHeightInches * pageDpi * 4 + ), + pageWidthInches * pageDpi, + pageHeightInches * pageDpi + ) + ); + + const layout = assertDefined(layouts[i]); + fs.writeFileSync(layoutFilePath, JSON.stringify(layout)); } - - // create directories for assets - fs.mkdirSync( - join(outputPath, `${CVR_BALLOT_IMAGES_SUBDIRECTORY}/${batchId}`), - { recursive: true } - ); - fs.mkdirSync( - join(outputPath, `${CVR_BALLOT_LAYOUTS_SUBDIRECTORY}/${batchId}`), - { recursive: true } - ); - - // write the image - await writeImageData( - join( - outputPath, - generateBallotAssetPath({ - ballotStyleId, - batchId, - precinctId, - pageNumber, - assetType: 'image', - }) - ), - createImageData( - new Uint8ClampedArray( - pageWidthInches * pageDpi * (pageHeightInches * pageDpi) * 4 - ), - pageWidthInches * pageDpi, - pageHeightInches * pageDpi - ) - ); - - // write the layout - const layout = find( - generateBallotPageLayouts(election, { - ballotStyleId, - precinctId, - electionHash, - ballotType: BallotType.Precinct, - isTestMode: testMode, - }), - (l) => l.metadata.pageNumber === pageNumber - ); - fs.writeFileSync( - join( - outputPath, - generateBallotAssetPath({ - ballotStyleId, - batchId, - precinctId, - pageNumber, - assetType: 'layout', - }) - ), - `${JSON.stringify(layout, undefined, 2)}\n` - ); } } + const castVoteRecordExportMetadata: CastVoteRecordExportMetadata = { + arePollsClosed: true, + castVoteRecordReportMetadata: reportMetadata, + castVoteRecordRootHash: + await computeCastVoteRecordRootHashFromScratch(outputPath), + }; + const metadataFileContents = JSON.stringify(castVoteRecordExportMetadata); + fs.writeFileSync(join(outputPath, 'metadata.json'), metadataFileContents); + + process.env['VX_MACHINE_TYPE'] = 'scan'; // Required by prepareSignatureFile + const signatureFile = await prepareSignatureFile({ + type: 'cast_vote_records', + context: 'export', + directoryName: basename(outputPath), + metadataFileContents, + }); + fs.writeFileSync( + join(parse(outputPath).dir, signatureFile.fileName), + signatureFile.fileContents + ); + stdout.write( `Wrote ${castVoteRecords.length} cast vote records to ${outputPath}\n` ); diff --git a/libs/cvr-fixture-generator/src/generate_cvrs.ts b/libs/cvr-fixture-generator/src/generate_cvrs.ts index 1cd3e2deee..b3cc163633 100644 --- a/libs/cvr-fixture-generator/src/generate_cvrs.ts +++ b/libs/cvr-fixture-generator/src/generate_cvrs.ts @@ -1,3 +1,4 @@ +import { v4 as uuid } from 'uuid'; import { buildCVRContestsFromVotes } from '@votingworks/backend'; import { iter, throwIllegalValue } from '@votingworks/basics'; import { @@ -198,7 +199,6 @@ export function* generateCvrs({ // not be realistic since they cannot currently be scanned. const bmdBallots = Boolean(!election.gridLayouts); - let castVoteRecordId = 0; for (const ballotStyle of ballotStyles) { const { precincts: precinctIds, id: ballotStyleId, partyId } = ballotStyle; // For each contest, determine all possible contest choices @@ -259,6 +259,7 @@ export function* generateCvrs({ // Add the generated vote combinations as CVRs for (const votes of voteConfigurations) { + const castVoteRecordId = uuid(); if (bmdBallots) { yield { '@type': 'CVR.CVR', @@ -288,8 +289,6 @@ export function* generateCvrs({ }, ], }; - - castVoteRecordId += 1; } else { // Since this is HMPB, we generate a CVR for each sheet (not fully supported yet) const contestsBySheet = arrangeContestsBySheet( @@ -314,18 +313,14 @@ export function* generateCvrs({ const sheetHasWriteIns = frontHasWriteIns || backHasWriteIns; const frontImageFileUri = `file:${generateBallotAssetPath({ - ballotStyleId: ballotStyle.id, - precinctId, - batchId, + castVoteRecordId: castVoteRecordId.toString(), assetType: 'image', - pageNumber: sheetIndex * 2 + 1, + frontOrBack: 'front', })}`; const backImageFileUri = `file:${generateBallotAssetPath({ - ballotStyleId: ballotStyle.id, - precinctId, - batchId, + castVoteRecordId: castVoteRecordId.toString(), assetType: 'image', - pageNumber: sheetIndex * 2 + 2, + frontOrBack: 'back', })}`; yield { @@ -371,21 +366,15 @@ export function* generateCvrs({ ? [ { '@type': 'CVR.ImageData', - Location: frontHasWriteIns - ? frontImageFileUri - : undefined, + Location: frontImageFileUri, }, { '@type': 'CVR.ImageData', - Location: backHasWriteIns - ? backImageFileUri - : undefined, + Location: backImageFileUri, }, ] : undefined, }; - - castVoteRecordId += 1; } } } diff --git a/libs/cvr-fixture-generator/src/utils.ts b/libs/cvr-fixture-generator/src/utils.ts index 3c8468ac79..1f25224f03 100644 --- a/libs/cvr-fixture-generator/src/utils.ts +++ b/libs/cvr-fixture-generator/src/utils.ts @@ -1,11 +1,8 @@ -import { - CVR_BALLOT_IMAGES_SUBDIRECTORY, - CVR_BALLOT_LAYOUTS_SUBDIRECTORY, -} from '@votingworks/backend'; import { assert, assertDefined } from '@votingworks/basics'; import { sha256 } from 'js-sha256'; import { BallotPageLayout, + BallotPaperSize, Contests, CVR, Election, @@ -15,6 +12,18 @@ import { VotesDict, } from '@votingworks/types'; +/** + * A mapping from ballot paper size to page height, in inches + */ +export const PAGE_HEIGHT_INCHES: Record = { + [BallotPaperSize.Letter]: 11, + [BallotPaperSize.Legal]: 14, + [BallotPaperSize.Custom17]: 17, + [BallotPaperSize.Custom18]: 18, + [BallotPaperSize.Custom21]: 21, + [BallotPaperSize.Custom22]: 22, +}; + /** * Generate all combinations of an array. * @param sourceArray - Array of input elements. @@ -139,34 +148,22 @@ export function filterVotesByContests( /** * Format of the image URIs used in generated fixtures. */ -export const IMAGE_URI_REGEX = new RegExp( - String.raw`file:${CVR_BALLOT_IMAGES_SUBDIRECTORY}\/(.+)\/(.+)__(.+)__(.+)\.jpg` -); +export const IMAGE_URI_REGEX = /file:(.+)-(front|back)\.jpg/; /** - * Generates the relative path, from the root of a cast vote record directory, - * to an asset specified by the parameters. + * Generates the path to a ballot asset, relative to an individual cast vote record directory. */ export function generateBallotAssetPath({ - ballotStyleId, - precinctId, - batchId, - pageNumber, + castVoteRecordId, assetType, + frontOrBack, }: { - ballotStyleId: string; - batchId: string; - precinctId: string; - pageNumber: number; + castVoteRecordId: string; assetType: 'image' | 'layout'; + frontOrBack: 'front' | 'back'; }): string { - return `${ - assetType === 'image' - ? CVR_BALLOT_IMAGES_SUBDIRECTORY - : CVR_BALLOT_LAYOUTS_SUBDIRECTORY - }/${batchId}/${ballotStyleId}__${precinctId}__${pageNumber}.${ - assetType === 'image' ? 'jpg' : 'layout.json' - }`; + const fileExtension = assetType === 'image' ? '.jpg' : '.layout.json'; + return `${castVoteRecordId}-${frontOrBack}${fileExtension}`; } /** diff --git a/libs/cvr-fixture-generator/tsconfig.build.json b/libs/cvr-fixture-generator/tsconfig.build.json index d5bbfc57e5..195e825b31 100644 --- a/libs/cvr-fixture-generator/tsconfig.build.json +++ b/libs/cvr-fixture-generator/tsconfig.build.json @@ -9,6 +9,7 @@ "declaration": true }, "references": [ + { "path": "../auth/tsconfig.build.json" }, { "path": "../basics/tsconfig.build.json" }, { "path": "../converter-nh-accuvote/tsconfig.build.json" }, { "path": "../eslint-plugin-vx/tsconfig.build.json" }, diff --git a/libs/cvr-fixture-generator/tsconfig.json b/libs/cvr-fixture-generator/tsconfig.json index df2a154ff0..ddcece836c 100644 --- a/libs/cvr-fixture-generator/tsconfig.json +++ b/libs/cvr-fixture-generator/tsconfig.json @@ -11,6 +11,7 @@ "skipLibCheck": true }, "references": [ + { "path": "../auth/tsconfig.build.json" }, { "path": "../basics/tsconfig.build.json" }, { "path": "../converter-nh-accuvote/tsconfig.build.json" }, { "path": "../eslint-plugin-vx/tsconfig.build.json" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2b9916a477..73eccdae76 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3605,6 +3605,9 @@ importers: libs/cvr-fixture-generator: dependencies: + '@votingworks/auth': + specifier: '*' + version: link:../auth '@votingworks/backend': specifier: '*' version: link:../backend @@ -3638,6 +3641,9 @@ importers: lodash.clonedeep: specifier: ^4.5.0 version: 4.5.0 + uuid: + specifier: ^9.0.1 + version: 9.0.1 yargs: specifier: ^17.5.1 version: 17.5.1 @@ -3654,6 +3660,9 @@ importers: '@types/tmp': specifier: ^0.2.3 version: 0.2.3 + '@types/uuid': + specifier: ^9.0.4 + version: 9.0.4 '@types/yargs': specifier: ^17.0.12 version: 17.0.12 @@ -10028,7 +10037,7 @@ packages: react-inspector: 6.0.1(react@18.2.0) telejson: 7.0.4 ts-dedent: 2.2.0 - uuid: 9.0.0 + uuid: 9.0.1 transitivePeerDependencies: - '@types/react' - '@types/react-dom' @@ -11685,6 +11694,10 @@ packages: resolution: {integrity: sha512-kNnC1GFBLuhImSnV7w4njQkUiJi0ZXUycu1rUaouPqiKlXkh77JKgdRnTAp1x5eBwcIwbtI+3otwzuIDEuDoxQ==} dev: true + /@types/uuid@9.0.4: + resolution: {integrity: sha512-zAuJWQflfx6dYJM62vna+Sn5aeSWhh3OB+wfUEACNcqUSc0AGc5JKl+ycL1vrH7frGTXhJchYjE1Hak8L819dA==} + dev: true + /@types/w3c-web-usb@1.0.6: resolution: {integrity: sha512-cSjhgrr8g4KbPnnijAr/KJDNKa/bBa+ixYkywFRvrhvi9n1WEl7yYbtRyzE6jqNQiSxxJxoAW3STaOQwJHndaw==} @@ -24820,6 +24833,11 @@ packages: /uuid@9.0.0: resolution: {integrity: sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==} hasBin: true + dev: false + + /uuid@9.0.1: + resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} + hasBin: true /v8-to-istanbul@9.1.0: resolution: {integrity: sha512-6z3GW9x8G1gd+JIIgQQQxXuiJtCXeAjp6RaPEPLv62mH3iPHPxV6W3robxtCzNErRo6ZwTmzWhsbNvjyEBKzKA==}