Add generate election package script (#5721)

* add generate-election-packages script with test when it needs regen

* make default latest metadata

* add version of electionPrimaryPrecinctSplits without ballot strings

* all new fixtures generated files

* fixtures build:resources

* export newly added fixtures for use in package generator test

* address pr comments

* fix refactor

apps/design/backend/src/store.ts

@@ -8,9 +8,9 @@ import {
   SystemSettings,
   safeParseSystemSettings,
   ElectionSerializationFormat,
-  BallotLanguageConfig,
   BallotLanguageConfigs,
   LanguageCode,
+  getBallotLanguageConfigs,
 } from '@votingworks/types';
 import { join } from 'node:path';
 import { v4 as uuid } from 'uuid';
@@ -26,11 +26,7 @@ export function getTempBallotLanguageConfigsForCert(): BallotLanguageConfigs {
   const translationsEnabled = isFeatureFlagEnabled(
     BooleanEnvironmentVariableName.ENABLE_CLOUD_TRANSLATION_AND_SPEECH_SYNTHESIS
   );
-  return translationsEnabled
-    ? Object.values(LanguageCode).map(
-        (l): BallotLanguageConfig => ({ languages: [l] })
-      )
-    : [{ languages: [LanguageCode.ENGLISH] }];
+  return getBallotLanguageConfigs(translationsEnabled);
 }
 
 export interface ElectionRecord {

apps/design/backend/src/worker/generate_election_package.ts

@@ -1,28 +1,25 @@
 import JsZip from 'jszip';
 import path from 'node:path';
 import {
-  BallotType,
   ElectionSerializationFormat,
   ElectionPackageFileName,
   ElectionPackageMetadata,
   Id,
   mergeUiStrings,
   Election,
   formatElectionHashes,
+  LATEST_METADATA,
 } from '@votingworks/types';
 
 import {
+  createElectionDefinitionForDefaultHmpbTemplate,
   createPlaywrightRenderer,
-  renderAllBallotsAndCreateElectionDefinition,
-  vxDefaultBallotTemplate,
 } from '@votingworks/hmpb';
 import { sha256 } from 'js-sha256';
 import { writeFile } from 'node:fs/promises';
 import {
-  translateAppStrings,
-  translateHmpbStrings,
-  extractAndTranslateElectionStrings,
   generateAudioIdsAndClips,
+  getAllStringsForElectionPackage,
 } from '@votingworks/backend';
 import { PORT } from '../globals';
 import { WorkerContext } from './context';
@@ -44,54 +41,36 @@ export async function generateElectionPackage(
 
   const zip = new JsZip();
 
-  const metadata: ElectionPackageMetadata = {
-    version: 'latest',
-  };
+  const metadata: ElectionPackageMetadata = LATEST_METADATA;
   zip.file(ElectionPackageFileName.METADATA, JSON.stringify(metadata, null, 2));
 
-  const appStrings = await translateAppStrings(
-    translator,
-    metadata.version,
-    ballotLanguageConfigs
-  );
+  const [appStrings, hmpbStrings, electionStrings] =
+    await getAllStringsForElectionPackage(
+      election,
+      translator,
+      ballotLanguageConfigs
+    );
+
   zip.file(
     ElectionPackageFileName.APP_STRINGS,
     JSON.stringify(appStrings, null, 2)
   );
 
-  const hmpbStrings = await translateHmpbStrings(
-    translator,
-    ballotLanguageConfigs
-  );
-  const electionStrings = await extractAndTranslateElectionStrings(
-    translator,
-    election,
-    ballotLanguageConfigs
-  );
   const ballotStrings = mergeUiStrings(electionStrings, hmpbStrings);
-
   const electionWithBallotStrings: Election = {
     ...election,
     ballotStrings,
   };
 
   const renderer = await createPlaywrightRenderer();
-  const { electionDefinition } =
-    await renderAllBallotsAndCreateElectionDefinition(
+  const electionDefinition =
+    await createElectionDefinitionForDefaultHmpbTemplate(
       renderer,
-      vxDefaultBallotTemplate,
-      // Each ballot style will have exactly one grid layout regardless of precinct, ballot type, or ballot mode
-      // So we just need to render a single ballot per ballot style to create the election definition
-      election.ballotStyles.map((ballotStyle) => ({
-        election: electionWithBallotStrings,
-        ballotStyleId: ballotStyle.id,
-        precinctId: ballotStyle.precincts[0],
-        ballotType: BallotType.Precinct,
-        ballotMode: 'test',
-      })),
+      electionWithBallotStrings,
       electionSerializationFormat
     );
   zip.file(ElectionPackageFileName.ELECTION, electionDefinition.electionData);
+
   // eslint-disable-next-line no-console
   renderer.cleanup().catch(console.error);
 

libs/backend/src/election_package/election_package_io.test.ts

@@ -4,6 +4,7 @@ import {
   ElectionPackageFileName,
   ElectionPackageMetadata,
   InsertedSmartCardAuth,
+  LATEST_METADATA,
   SystemSettings,
   UiStringAudioClips,
   UiStringAudioIdsPackage,
@@ -298,7 +299,7 @@ test('readElectionPackageFromFile reads metadata', async () => {
   const { electionDefinition } =
     electionGridLayoutNewHampshireTestBallotFixtures;
   const { electionData } = electionDefinition;
-  const metadata: ElectionPackageMetadata = { version: 'latest' };
+  const metadata: ElectionPackageMetadata = LATEST_METADATA;
 
   const pkg = await zipFile({
     [ElectionPackageFileName.ELECTION]: electionData,

libs/backend/src/language_and_audio/audio.ts

@@ -10,7 +10,7 @@ import {
   isFeatureFlagEnabled,
   BooleanEnvironmentVariableName,
 } from '@votingworks/utils';
-import { GoogleCloudSpeechSynthesizer } from './speech_synthesizer';
+import { SpeechSynthesizer } from './speech_synthesizer';
 import {
   forEachUiString,
   prepareTextForSpeechSynthesis,
@@ -32,7 +32,7 @@ export function generateAudioIdsAndClips({
 }: {
   appStrings: UiStringsPackage;
   electionStrings: UiStringsPackage;
-  speechSynthesizer: GoogleCloudSpeechSynthesizer;
+  speechSynthesizer: SpeechSynthesizer;
 }): {
   uiStringAudioIds: UiStringAudioIdsPackage;
   uiStringAudioClips: NodeJS.ReadableStream;

libs/backend/src/language_and_audio/election_package_string.test.ts

@@ -0,0 +1,60 @@
+import { electionPrimaryPrecinctSplitsFixtures } from '@votingworks/fixtures';
+import { LanguageCode, BallotLanguageConfigs } from '@votingworks/types';
+import { assert } from '@votingworks/basics';
+import { getAllStringsForElectionPackage } from './election_package_strings';
+import { GoogleCloudTranslator } from './translator';
+import { MockGoogleCloudTranslationClient } from './test_utils';
+
+const allBallotLanguages: BallotLanguageConfigs = [
+  {
+    languages: [
+      LanguageCode.ENGLISH,
+      LanguageCode.CHINESE_SIMPLIFIED,
+      LanguageCode.CHINESE_TRADITIONAL,
+      LanguageCode.SPANISH,
+    ],
+  },
+];
+
+describe('getAllStringsForElectionPackage', () => {
+  it('should extract and translate election strings correctly for english only', async () => {
+    const translationClient = new MockGoogleCloudTranslationClient();
+    const mockTranslator = new GoogleCloudTranslator({ translationClient });
+    const [appStrings, hmpbStrings, electionStrings] =
+      await getAllStringsForElectionPackage(
+        electionPrimaryPrecinctSplitsFixtures.election,
+        mockTranslator,
+        allBallotLanguages
+      );
+
+    expect(appStrings).toBeDefined();
+    expect(Object.keys(appStrings)).toEqual([
+      LanguageCode.ENGLISH,
+      LanguageCode.CHINESE_SIMPLIFIED,
+      LanguageCode.CHINESE_TRADITIONAL,
+      LanguageCode.SPANISH,
+    ]);
+    assert(appStrings[LanguageCode.ENGLISH]);
+    expect(Object.keys(appStrings[LanguageCode.ENGLISH])).toHaveLength(427);
+
+    expect(hmpbStrings).toBeDefined();
+    expect(Object.keys(hmpbStrings)).toEqual([
+      LanguageCode.ENGLISH,
+      LanguageCode.CHINESE_SIMPLIFIED,
+      LanguageCode.CHINESE_TRADITIONAL,
+      LanguageCode.SPANISH,
+    ]);
+    assert(hmpbStrings[LanguageCode.ENGLISH]);
+    expect(Object.keys(hmpbStrings[LanguageCode.ENGLISH])).toHaveLength(30);
+
+    expect(electionStrings).toBeDefined();
+    expect(Object.keys(electionStrings)).toEqual([
+      LanguageCode.ENGLISH,
+      LanguageCode.CHINESE_SIMPLIFIED,
+      LanguageCode.CHINESE_TRADITIONAL,
+      LanguageCode.SPANISH,
+    ]);
+    assert(electionStrings[LanguageCode.ENGLISH]);
+    expect(Object.keys(electionStrings[LanguageCode.ENGLISH])).toHaveLength(14);
+  });
+});

libs/backend/src/language_and_audio/election_package_strings.ts

@@ -0,0 +1,36 @@
+import {
+  BallotLanguageConfigs,
+  Election,
+  UiStringsPackage,
+} from '@votingworks/types';
+import { GoogleCloudTranslator } from './translator';
+import { translateAppStrings } from './app_strings';
+import { translateHmpbStrings } from './ballot_strings';
+import { extractAndTranslateElectionStrings } from './election_strings';
+
+/**
+ * Helper function to generate all necessary strings used in an election package.
+ * Returns three packages of strings: app strings, HMPB strings, and election strings.
+ */
+export async function getAllStringsForElectionPackage(
+  election: Election,
+  translator: GoogleCloudTranslator,
+  ballotLanguageConfigs: BallotLanguageConfigs
+): Promise<[UiStringsPackage, UiStringsPackage, UiStringsPackage]> {
+  const appStrings = await translateAppStrings(
+    translator,
+    'latest',
+    ballotLanguageConfigs
+  );
+  const hmpbStrings = await translateHmpbStrings(
+    translator,
+    ballotLanguageConfigs
+  );
+  const electionStrings = await extractAndTranslateElectionStrings(
+    translator,
+    election,
+    ballotLanguageConfigs
+  );
+
+  return [appStrings, hmpbStrings, electionStrings];
+}

libs/backend/src/language_and_audio/index.ts

@@ -3,6 +3,7 @@ export * from './app_strings';
 export * from './ballot_strings';
 export * from './audio';
 export * from './election_strings';
+export * from './election_package_strings';
 export * from './hmpb_strings';
 export * from './speech_synthesizer';
 export * from './test_utils';

libs/fixture-generators/README.md

@@ -77,3 +77,33 @@ are optional and have default values.
 ```bash
 ./bin/generate-election config.json > election.json
 ```
+
+## Election Package Generator
+
+A command-line tool for generating election packages and elections with grid
+layouts and translations. Follow the instructions to setup Google Cloud account
+authentication [here](/../backend/src/language_and_audio/README.md) before
+using.
+
+```bash
+./bin/generate-election-package -e path/to/base-election-definition.json -o path/to/output-directory
+```
+
+To generate an election.json and election-package file in the specified output
+directory with gridLayouts and all necessary strings from the base election
+provided. If --isMultiLanguage is specified then the strings will include
+translations for all languages. If --priorElectionPackage is specified that
+election package will be used as a cache for translations before querying google
+cloud.
+
+## Election Packages Generator
+
+Wrapper script to regenerate election packages for all configured fixtures.
+
+```bash
+pnpm generate-election-packages
+```
+
+Run with FORCE_RETRANSLATE=1 to make new translations generate for all election
+packages. Note you will need to run pnpm build:resources && pnpm build in
+libs/fixtures after running this to register the new fixtures.

libs/fixture-generators/bin/generate-election-package

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+require('esbuild-runner').install({
+  type: 'transform',
+});
+
+require('../src/cli/generate-election-package')
+  .main(process.argv, {
+    stdin: process.stdin,
+    stdout: process.stdout,
+    stderr: process.stderr,
+  })
+  .catch((e) => {
+    console.error(e);
+    process.exit(1);
+  });

libs/fixture-generators/bin/regenerate-election-packages

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# Generates the saved election package fixtures in libs/fixtures
+# By default it will reuse the translations and audio from previous runs if new strings
+# are not detected to be translated. To force re-translation, set the environment variable
+# FORCE_RETRANSLATE to true.
+export NODE_ENV=development
+
+# Check if FORCE_RETRANSLATE is set to true
+if [ -z "$FORCE_RETRANSLATE" ]; then
+  ./bin/generate-election-package -e ../fixtures/data/electionPrimaryPrecinctSplits/electionBase.json -o ../fixtures/data/electionPrimaryPrecinctSplits/ -p ../fixtures/data/electionPrimaryPrecinctSplits/election-package-default-system-settings.zip --isMultiLanguage
+else
+  ./bin/generate-election-package -e ../fixtures/data/electionPrimaryPrecinctSplits/electionBase.json -o ../fixtures/data/electionPrimaryPrecinctSplits/ --isMultiLanguage
+fi
+
+echo
+echo "Note: You need to run \`pnpm build:resources && pnpm build\` in libs/fixtures for the new fixtures to register"

libs/fixture-generators/jest.config.js

@@ -12,6 +12,9 @@ module.exports = {
     // that it creates a valid election (which it checks when generating the
     // election).
     '!src/generate-election/*',
+    // The test for generate-election-package checks that the fixtures do not need to
+    // be updated which inherently checks that the package is generated correctly.
+    '!src/generate-election-package/*',
   ],
   coverageThreshold: {
     global: {

libs/fixture-generators/package.json

@@ -15,6 +15,7 @@
     "clean": "pnpm --filter $npm_package_name... clean:self",
     "clean:self": "rm -rf build && tsc --build --clean tsconfig.build.json",
     "generate-cvr-fixtures": "./bin/generate-cvr-fixtures",
+    "generate-election-packages": "./bin/regenerate-election-packages",
     "lint": "pnpm type-check && eslint .",
     "lint:fix": "pnpm type-check && eslint . --fix",
     "test": "is-ci test:ci test:watch",
@@ -29,18 +30,22 @@
     "@votingworks/basics": "workspace:*",
     "@votingworks/fixtures": "workspace:*",
     "@votingworks/fs": "workspace:*",
+    "@votingworks/hmpb": "workspace:*",
     "@votingworks/image-utils": "workspace:*",
     "@votingworks/types": "workspace:*",
     "@votingworks/utils": "workspace:*",
+    "debug": "4.3.4",
     "esbuild": "0.21.2",
     "esbuild-runner": "2.2.2",
     "js-sha256": "^0.9.0",
+    "jszip": "^3.9.1",
     "nanoid": "^3.3.7",
     "uuid": "9.0.1",
     "yargs": "17.7.1",
     "zod": "3.23.5"
   },
   "devDependencies": {
+    "@types/debug": "4.1.8",
     "@types/jest": "^29.5.3",
     "@types/node": "20.16.0",
     "@types/tmp": "0.2.4",
@@ -55,5 +60,8 @@
     "tmp": "^0.2.1",
     "ts-jest": "29.1.1"
   },
+  "engines": {
+    "node": ">= 12"
+  },
   "packageManager": "[email protected]"
-}
+}