Update VxAdmin to support importing new CVR export format

apps/admin/backend/src/app.ts

@@ -32,9 +32,11 @@ import { createReadStream, createWriteStream, promises as fs, Stats } from 'fs';
 import { basename, dirname, join } from 'path';
 import {
   BALLOT_PACKAGE_FOLDER,
+  BooleanEnvironmentVariableName,
   CAST_VOTE_RECORD_REPORT_FILENAME,
   generateFilenameForBallotExportPackage,
   groupMapToGroupList,
+  isFeatureFlagEnabled,
   isIntegrationTest,
   parseCastVoteRecordReportDirectoryName,
 } from '@votingworks/utils';
@@ -48,6 +50,7 @@ import {
   CvrFileMode,
   ElectionRecord,
   ExportDataResult,
+  ImportCastVoteRecordsError,
   ManualResultsIdentifier,
   ManualResultsMetadataRecord,
   ManualResultsRecord,
@@ -68,7 +71,7 @@ import {
   addCastVoteRecordReport,
   getAddCastVoteRecordReportErrorMessage,
   listCastVoteRecordFilesOnUsb,
-} from './cvr_files';
+} from './legacy_cast_vote_records';
 import { getMachineConfig } from './machine_config';
 import {
   getWriteInAdjudicationContext,
@@ -86,6 +89,7 @@ import { getOverallElectionWriteInSummary } from './tabulation/write_ins';
 import { rootDebug } from './util/debug';
 import { tabulateTallyReportResults } from './tabulation/tally_reports';
 import { buildExporter } from './util/exporter';
+import { importCastVoteRecords } from './cast_vote_records';
 
 const debug = rootDebug.extend('app');
 
@@ -404,9 +408,39 @@ function buildApi({
     }): Promise<
       Result<
         CvrFileImportInfo,
-        AddCastVoteRecordReportError & { message: string }
+        | (AddCastVoteRecordReportError & { message: string })
+        | ImportCastVoteRecordsError
       >
     > {
+      /* c8 ignore start */
+      if (
+        isFeatureFlagEnabled(
+          BooleanEnvironmentVariableName.ENABLE_CONTINUOUS_EXPORT
+        )
+      ) {
+        const userRole = assertDefined(await getUserRole());
+        const importResult = await importCastVoteRecords(store, input.path);
+        if (importResult.isErr()) {
+          await logger.log(LogEventId.CvrLoaded, userRole, {
+            disposition: 'failure',
+            errorDetails: JSON.stringify(importResult.err()),
+            errorType: importResult.err().type,
+            exportDirectoryPath: input.path,
+            result: 'Cast vote records not imported, error shown to user.',
+          });
+        } else {
+          await logger.log(LogEventId.CvrLoaded, userRole, {
+            disposition: 'success',
+            exportDirectoryPath: input.path,
+            numberOfBallotsImported: importResult.ok().newlyAdded,
+            numberOfDuplicateBallotsIgnored: importResult.ok().alreadyPresent,
+            result: 'Cast vote records imported.',
+          });
+        }
+        return importResult;
+      }
+      /* c8 ignore stop */
+
       const userRole = assertDefined(await getUserRole());
       const { path: inputPath } = input;
       // the path passed to the backend may be for the report directory or the

apps/admin/backend/src/cast_vote_records.ts

@@ -0,0 +1,283 @@
+/* c8 ignore start */
+import * as fs from 'fs/promises';
+import { sha256 } from 'js-sha256';
+import path from 'path';
+import { v4 as uuid } from 'uuid';
+import { isTestReport, readCastVoteRecordExport } from '@votingworks/backend';
+import { assert, assertDefined, err, ok, Result } from '@votingworks/basics';
+import {
+  BallotId,
+  BallotPageLayoutSchema,
+  CVR,
+  ElectionDefinition,
+  getContests,
+  safeParseJson,
+} from '@votingworks/types';
+import {
+  BooleanEnvironmentVariableName,
+  castVoteRecordHasValidContestReferences,
+  convertCastVoteRecordVotesToTabulationVotes,
+  getBallotStyleById,
+  getPrecinctById,
+  isFeatureFlagEnabled,
+} from '@votingworks/utils';
+
+import { Store } from './store';
+import {
+  CastVoteRecordElectionDefinitionValidationError,
+  CvrFileImportInfo,
+  CvrFileMode,
+  ImportCastVoteRecordsError,
+} from './types';
+
+/**
+ * Validates that the fields in a cast vote record and the election definition correspond
+ */
+function validateCastVoteRecordAgainstElectionDefinition(
+  castVoteRecord: CVR.CVR,
+  electionDefinition: ElectionDefinition
+): Result<void, CastVoteRecordElectionDefinitionValidationError> {
+  function wrapError(
+    error: Omit<CastVoteRecordElectionDefinitionValidationError, 'type'>
+  ): Result<void, CastVoteRecordElectionDefinitionValidationError> {
+    return err({ ...error, type: 'invalid-cast-vote-record' });
+  }
+
+  const { election, electionHash } = electionDefinition;
+
+  if (
+    castVoteRecord.ElectionId !== electionHash &&
+    !isFeatureFlagEnabled(
+      BooleanEnvironmentVariableName.SKIP_CVR_ELECTION_HASH_CHECK
+    )
+  ) {
+    return wrapError({ subType: 'election-mismatch' });
+  }
+
+  const precinct = getPrecinctById(
+    electionDefinition,
+    castVoteRecord.BallotStyleUnitId
+  );
+  if (!precinct) {
+    return wrapError({ subType: 'precinct-not-found' });
+  }
+
+  const ballotStyle = getBallotStyleById(
+    electionDefinition,
+    castVoteRecord.BallotStyleId
+  );
+  if (!ballotStyle) {
+    return wrapError({ subType: 'ballot-style-not-found' });
+  }
+
+  const contestValidationResult = castVoteRecordHasValidContestReferences(
+    castVoteRecord,
+    getContests({ ballotStyle, election })
+  );
+  if (contestValidationResult.isErr()) {
+    return wrapError({ subType: contestValidationResult.err() });
+  }
+
+  return ok();
+}
+
+/**
+ * Imports cast vote records given a cast vote record export directory path
+ */
+export async function importCastVoteRecords(
+  store: Store,
+  exportDirectoryPath: string
+): Promise<Result<CvrFileImportInfo, ImportCastVoteRecordsError>> {
+  const electionId = assertDefined(store.getCurrentElectionId());
+  const { electionDefinition } = assertDefined(store.getElection(electionId));
+
+  const readResult = await readCastVoteRecordExport(exportDirectoryPath);
+  if (readResult.isErr()) {
+    return readResult;
+  }
+  const { castVoteRecordExportMetadata, castVoteRecords } = readResult.ok();
+  const { castVoteRecordReportMetadata } = castVoteRecordExportMetadata;
+
+  const exportDirectoryName = path.basename(exportDirectoryPath);
+  // Hashing the export metadata, which includes a root hash of all the individual cast vote
+  // records, gives us a complete hash of the entire export
+  const exportHash = sha256(JSON.stringify(castVoteRecordExportMetadata));
+  const exportedTimestamp = castVoteRecordReportMetadata.GeneratedDate;
+
+  // Ensure that the records to be imported match the mode (test vs. official) of previously
+  // imported records
+  const mode: CvrFileMode = isTestReport(castVoteRecordReportMetadata)
+    ? 'test'
+    : 'official';
+  const currentMode = store.getCurrentCvrFileModeForElection(electionId);
+  if (currentMode !== 'unlocked' && mode !== currentMode) {
+    return err({ type: 'invalid-mode', currentMode });
+  }
+
+  const existingImportId = store.getCastVoteRecordFileByHash(
+    electionId,
+    exportHash
+  );
+  if (existingImportId) {
+    return ok({
+      id: existingImportId,
+      alreadyPresent: store.getCastVoteRecordCountByFileId(existingImportId),
+      exportedTimestamp,
+      fileMode: mode,
+      fileName: exportDirectoryName,
+      newlyAdded: 0,
+      wasExistingFile: true,
+    });
+  }
+
+  return await store.withTransaction(async () => {
+    const scannerIds = new Set<string>();
+    for (const vxBatch of castVoteRecordReportMetadata.vxBatch) {
+      store.addScannerBatch({
+        batchId: vxBatch['@id'],
+        electionId,
+        label: vxBatch.BatchLabel,
+        scannerId: vxBatch.CreatingDeviceId,
+      });
+      scannerIds.add(vxBatch.CreatingDeviceId);
+    }
+
+    // Create a top-level record for the import
+    const importId = uuid();
+    store.addCastVoteRecordFileRecord({
+      id: importId,
+      electionId,
+      exportedTimestamp,
+      filename: exportDirectoryName,
+      isTestMode: isTestReport(castVoteRecordReportMetadata),
+      scannerIds,
+      sha256Hash: exportHash,
+    });
+
+    let castVoteRecordIndex = 0;
+    let newlyAdded = 0;
+    let alreadyPresent = 0;
+    const precinctIds = new Set<string>();
+    for await (const castVoteRecordResult of castVoteRecords) {
+      if (castVoteRecordResult.isErr()) {
+        return err({
+          ...castVoteRecordResult.err(),
+          index: castVoteRecordIndex,
+        });
+      }
+      const {
+        castVoteRecord,
+        castVoteRecordBallotSheetId,
+        castVoteRecordCurrentSnapshot,
+        castVoteRecordWriteIns,
+        referencedFiles,
+      } = castVoteRecordResult.ok();
+
+      const validationResult = validateCastVoteRecordAgainstElectionDefinition(
+        castVoteRecord,
+        electionDefinition
+      );
+      if (validationResult.isErr()) {
+        return err({ ...validationResult.err(), index: castVoteRecordIndex });
+      }
+
+      // Add an individual cast vote record to the import
+      const votes = convertCastVoteRecordVotesToTabulationVotes(
+        castVoteRecordCurrentSnapshot
+      );
+      const addCastVoteRecordResult = store.addCastVoteRecordFileEntry({
+        ballotId: castVoteRecord.UniqueId as BallotId,
+        cvr: {
+          ballotStyleId: castVoteRecord.BallotStyleId,
+          batchId: castVoteRecord.BatchId,
+          card: castVoteRecordBallotSheetId
+            ? { type: 'hmpb', sheetNumber: castVoteRecordBallotSheetId }
+            : { type: 'bmd' },
+          precinctId: castVoteRecord.BallotStyleUnitId,
+          votes,
+          votingMethod: castVoteRecord.vxBallotType,
+        },
+        cvrFileId: importId,
+        electionId,
+      });
+      if (addCastVoteRecordResult.isErr()) {
+        return err({
+          ...addCastVoteRecordResult.err(),
+          index: castVoteRecordIndex,
+        });
+      }
+      const { cvrId: castVoteRecordId, isNew: isCastVoteRecordNew } =
+        addCastVoteRecordResult.ok();
+
+      if (isCastVoteRecordNew) {
+        const hmpbCastVoteRecordWriteIns = castVoteRecordWriteIns.filter(
+          (castVoteRecordWriteIn) => castVoteRecordWriteIn.side
+        );
+        if (hmpbCastVoteRecordWriteIns.length > 0) {
+          // Guaranteed to exist given validation in readCastVoteRecordExport
+          assert(referencedFiles !== undefined);
+
+          for (const i of [0, 1] as const) {
+            const imageData = await fs.readFile(
+              referencedFiles.imageFilePaths[i]
+            );
+            const parseLayoutResult = safeParseJson(
+              await fs.readFile(referencedFiles.layoutFilePaths[i], 'utf8'),
+              BallotPageLayoutSchema
+            );
+            if (parseLayoutResult.isErr()) {
+              return err({
+                type: 'invalid-cast-vote-record',
+                subType: 'layout-parse-error',
+                index: castVoteRecordIndex,
+              });
+            }
+            store.addBallotImage({
+              cvrId: castVoteRecordId,
+              imageData,
+              pageLayout: parseLayoutResult.ok(),
+              side: (['front', 'back'] as const)[i],
+            });
+          }
+
+          for (const hmpbCastVoteRecordWriteIn of hmpbCastVoteRecordWriteIns) {
+            store.addWriteIn({
+              castVoteRecordId,
+              contestId: hmpbCastVoteRecordWriteIn.contestId,
+              electionId,
+              optionId: hmpbCastVoteRecordWriteIn.optionId,
+              side: assertDefined(hmpbCastVoteRecordWriteIn.side),
+            });
+          }
+        }
+      }
+
+      if (isCastVoteRecordNew) {
+        newlyAdded += 1;
+      } else {
+        alreadyPresent += 1;
+      }
+      precinctIds.add(castVoteRecord.BallotStyleUnitId);
+
+      castVoteRecordIndex += 1;
+    }
+
+    // TODO: Calculate the precinct list before iterating through cast vote records, once there is
+    // only one geopolitical unit per batch
+    store.updateCastVoteRecordFileRecord({
+      id: importId,
+      precinctIds,
+    });
+
+    return ok({
+      id: importId,
+      alreadyPresent,
+      exportedTimestamp,
+      fileMode: mode,
+      fileName: exportDirectoryName,
+      newlyAdded,
+      wasExistingFile: false,
+    });
+  });
+}
+/* c8 ignore stop */

apps/admin/backend/src/cvr_files.test.ts → apps/admin/backend/src/legacy_cast_vote_records.test.ts

@@ -6,7 +6,7 @@ import { createMockUsbDrive } from '@votingworks/usb-drive';
 import {
   listCastVoteRecordFilesOnUsb,
   validateCastVoteRecord,
-} from './cvr_files';
+} from './legacy_cast_vote_records';
 
 const electionDefinition = electionTwoPartyPrimaryDefinition;
 const file = Buffer.from([]);
@@ -361,7 +361,7 @@ describe('validateCastVoteRecord', () => {
       reportBatchIds: ['batch-1'],
     });
 
-    expect(result.err()).toEqual('invalid-contest');
+    expect(result.err()).toEqual('contest-not-found');
   });
 
   test('error on invalid contest option id', () => {
@@ -392,7 +392,7 @@ describe('validateCastVoteRecord', () => {
       reportBatchIds: ['batch-1'],
     });
 
-    expect(result.err()).toEqual('invalid-contest-option');
+    expect(result.err()).toEqual('contest-option-not-found');
   });
 
   test('error on unknown ballot image', () => {