Merge remote-tracking branch 'origin/develop' into feature/issue-679_…

…confirm-num-ballots

.github/workflows/release.yml

@@ -3,6 +3,8 @@ name: "Generate Releases"
 on:
   release:
     types: [ published ]
+  schedule:
+    - cron: '0 12 1,15 * *'  # On the 1st and 15th of the month at noon
 ## To test this workflow without creating a release, uncomment the following and add a branch name (making sure "push"
 ##  is at the same indent level as "release":
 #  push:
@@ -136,7 +138,7 @@ jobs:
 
       - name: "Upload binaries to release"
         uses: svenstaro/upload-release-action@v2
-        if: github.event_name == 'release'
+        if: github.event_name == 'release' || github.event_name == 'schedule'
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           file: build/${{ steps.basefn.outputs.FILEPATH }}*

src/main/java/network/brightspots/rcv/CastVoteRecord.java

@@ -89,8 +89,12 @@ class CastVoteRecord {
   }
 
   CastVoteRecord(
-      String computedId, String suppliedId, String precinct, List<Pair<Integer, String>> rankings) {
-    this(null, null, null, suppliedId, computedId, precinct, null, rankings);
+      String computedId,
+      String suppliedId,
+      String precinct,
+      String batchId,
+      List<Pair<Integer, String>> rankings) {
+    this(null, null, batchId, suppliedId, computedId, precinct, null, rankings);
   }
 
   String getContestId() {
@@ -101,12 +105,11 @@ String getTabulatorId() {
     return tabulatorId;
   }
 
-  String getBatchId() {
-    return batchId;
-  }
-
-  String getPrecinct() {
-    return precinct;
+  String getSlice(ContestConfig.TabulateBySlice slice) {
+    return switch (slice) {
+      case BATCH -> batchId;
+      case PRECINCT -> precinct;
+    };
   }
 
   String getPrecinctPortion() {

src/main/java/network/brightspots/rcv/CommonDataFormatReader.java

@@ -315,8 +315,8 @@ void parseXml(List<CastVoteRecord> castVoteRecords) throws CvrParseException, IO
 
         String computedCastVoteRecordId = String.format("%s(%d)", fileName, ++cvrIndex);
         // create the new CastVoteRecord
-        CastVoteRecord newRecord =
-            new CastVoteRecord(computedCastVoteRecordId, cvr.UniqueId, precinctId, rankings);
+        CastVoteRecord newRecord = new CastVoteRecord(
+            computedCastVoteRecordId, cvr.UniqueId, precinctId, cvr.BatchSequenceId, rankings);
         castVoteRecords.add(newRecord);
 
         // provide some user feedback on the CVR count
@@ -524,10 +524,11 @@ void parseJson(List<CastVoteRecord> castVoteRecords) throws CvrParseException {
       }
 
       String ballotId = (String) cvr.get("BallotPrePrintedId");
+      String batchId = (String) cvr.get("BatchSequenceId");
       String computedCastVoteRecordId = String.format("%s(%d)", fileName, ++cvrIndex);
       // create the new CastVoteRecord
       CastVoteRecord newRecord =
-          new CastVoteRecord(computedCastVoteRecordId, ballotId, precinctId, rankings);
+          new CastVoteRecord(computedCastVoteRecordId, ballotId, precinctId, batchId, rankings);
       castVoteRecords.add(newRecord);
       // provide some user feedback on the CVR count
       if (castVoteRecords.size() % 50000 == 0) {

src/main/java/network/brightspots/rcv/ContestConfig.java

@@ -32,6 +32,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import network.brightspots.rcv.RawContestConfig.Candidate;
 import network.brightspots.rcv.RawContestConfig.CvrSource;
@@ -44,6 +45,7 @@ class ContestConfig {
   // If any booleans are unspecified in config file, they should default to false no matter what
   static final String AUTOMATED_TEST_VERSION = "TEST";
   static final String SUGGESTED_OUTPUT_DIRECTORY = "output";
+  static final boolean SUGGESTED_TABULATE_BY_BATCH = false;
   static final boolean SUGGESTED_TABULATE_BY_PRECINCT = false;
   static final boolean SUGGESTED_GENERATE_CDF_JSON = false;
   static final boolean SUGGESTED_CANDIDATE_EXCLUDED = false;
@@ -228,6 +230,14 @@ && stringAlreadyInUseElsewhereInSource(
               source.getFilePath())) {
             validationErrors.add(ValidationError.CVR_PRECINCT_COLUMN_UNEXPECTEDLY_DEFINED);
           }
+
+          if (fieldIsDefinedButShouldNotBeForProvider(
+                  source.getBatchColumnIndex(),
+                  "batchColumnIndex",
+                  provider,
+                  source.getFilePath())) {
+            validationErrors.add(ValidationError.CVR_BATCH_COLUMN_UNEXPECTEDLY_DEFINED);
+          }
         }
 
         // See the config file documentation for an explanation of this regex
@@ -284,6 +294,14 @@ && stringAlreadyInUseElsewhereInSource(
           validationErrors.add(ValidationError.CVR_PRECINCT_COLUMN_UNEXPECTEDLY_DEFINED);
         }
 
+        if (fieldIsDefinedButShouldNotBeForProvider(
+                source.getBatchColumnIndex(),
+                "batchColumnIndex",
+                provider,
+                source.getFilePath())) {
+          validationErrors.add(ValidationError.CVR_BATCH_COLUMN_UNEXPECTEDLY_DEFINED);
+        }
+
         if (fieldIsDefinedButShouldNotBeForProvider(
             source.getOvervoteDelimiter(), "overvoteDelimiter", provider, source.getFilePath())) {
           validationErrors.add(ValidationError.CVR_OVERVOTE_DELIMITER_UNEXPECTEDLY_DEFINED);
@@ -599,16 +617,26 @@ && getOvervoteRule() != Tabulator.OvervoteRule.ALWAYS_SKIP_TO_NEXT_RANK) {
 
         if (getProvider(source) == Provider.CDF) {
           // Perform CDF checks
-          if (isTabulateByPrecinctEnabled()) {
-            validationErrors.add(ValidationError.CVR_CDF_TABULATE_BY_PRECINCT_DISAGREEMENT);
-            Logger.severe("tabulateByPrecinct may not be used with CDF files.");
+          for (TabulateBySlice tabulateBySlice : enabledSlices()) {
+            validationErrors.add(ValidationError.CVR_CDF_TABULATE_BY_DISAGREEMENT);
+            Logger.severe("Tabulate-by-%s has not yet been implemented for CDF files.",
+                    tabulateBySlice);
           }
         } else if (getProvider(source) == Provider.ESS) {
           // Perform ES&S checks
-          if (isNullOrBlank(source.getPrecinctColumnIndex()) && isTabulateByPrecinctEnabled()) {
+          if (isNullOrBlank(source.getPrecinctColumnIndex())
+              && isTabulateByEnabled(TabulateBySlice.PRECINCT)) {
             validationErrors.add(ValidationError.CVR_TABULATE_BY_PRECINCT_REQUIRES_PRECINCT_COLUMN);
             Logger.severe(
-                "precinctColumnIndex is required when tabulateByPrecinct is enabled: %s", cvrPath);
+                "precinctColumnIndex is required when tabulateByPrecinct is enabled: %s",
+                cvrPath);
+          }
+          if (isNullOrBlank(source.getBatchColumnIndex())
+              && isTabulateByEnabled(TabulateBySlice.BATCH)) {
+            validationErrors.add(ValidationError.CVR_TABULATE_BY_PRECINCT_REQUIRES_BATCH_COLUMN);
+            Logger.severe(
+                    "batchColumnIndex is required when tabulateByBatch is enabled: %s",
+                    cvrPath);
           }
           if (isNullOrBlank(source.getOvervoteDelimiter())
               && getOvervoteRule() == OvervoteRule.EXHAUST_IF_MULTIPLE_CONTINUING) {
@@ -959,8 +987,17 @@ String getContestDate() {
     return rawConfig.outputSettings.contestDate;
   }
 
-  boolean isTabulateByPrecinctEnabled() {
-    return rawConfig.outputSettings.tabulateByPrecinct;
+  boolean isTabulateByEnabled(TabulateBySlice slice) {
+    return switch (slice) {
+      case PRECINCT -> rawConfig.outputSettings.tabulateByPrecinct;
+      case BATCH -> rawConfig.outputSettings.tabulateByBatch;
+    };
+  }
+
+  List<TabulateBySlice> enabledSlices() {
+    return Arrays.stream(TabulateBySlice.values())
+            .filter(this::isTabulateByEnabled)
+            .collect(Collectors.toList());
   }
 
   boolean isGenerateCdfJsonEnabled() {
@@ -1194,6 +1231,7 @@ enum ValidationError {
     CVR_FIRST_VOTE_COLUMN_INVALID,
     CVR_FIRST_VOTE_ROW_INVALID,
     CVR_ID_COLUMN_INVALID,
+    CVR_BATCH_COLUMN_INVALID,
     CVR_PRECINCT_COLUMN_INVALID,
     CVR_OVERVOTE_DELIMITER_INVALID,
     CVR_CDF_FILE_PATH_INVALID,
@@ -1202,7 +1240,8 @@ enum ValidationError {
     CVR_DUPLICATE_FILE_PATHS,
     CVR_FILE_PATH_INVALID,
     CVR_OVERVOTE_LABEL_OVERVOTE_RULE_MISMATCH,
-    CVR_CDF_TABULATE_BY_PRECINCT_DISAGREEMENT,
+    CVR_CDF_TABULATE_BY_DISAGREEMENT,
+    CVR_TABULATE_BY_PRECINCT_REQUIRES_BATCH_COLUMN,
     CVR_TABULATE_BY_PRECINCT_REQUIRES_PRECINCT_COLUMN,
     CVR_OVERVOTE_DELIMITER_AND_LABEL_BOTH_SUPPLIED,
     CVR_OVERVOTE_DELIMITER_MISSING,
@@ -1211,6 +1250,7 @@ enum ValidationError {
     CVR_FIRST_VOTE_UNEXPECTEDLY_DEFINED,
     CVR_FIRST_VOTE_ROW_UNEXPECTEDLY_DEFINED,
     CVR_ID_COLUMN_UNEXPECTEDLY_DEFINED,
+    CVR_BATCH_COLUMN_UNEXPECTEDLY_DEFINED,
     CVR_PRECINCT_COLUMN_UNEXPECTEDLY_DEFINED,
     CVR_SKIPPED_RANK_LABEL_UNEXPECTEDLY_DEFINED,
     CVR_CONTEST_ID_UNEXPECTEDLY_DEFINED,
@@ -1291,5 +1331,25 @@ public String getInternalLabel() {
     }
   }
 
+  enum TabulateBySlice {
+    PRECINCT("Precinct"),
+    BATCH("Batch");
+
+    private final String label;
+
+    TabulateBySlice(String label) {
+      this.label = label;
+    }
+
+    @Override
+    public String toString() {
+      return label;
+    }
+
+    public String toLowerString() {
+      return label.toLowerCase();
+    }
+  }
+
   static class UnrecognizedProviderException extends Exception {}
 }

src/main/java/network/brightspots/rcv/CsvCvrReader.java

@@ -107,8 +107,12 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords)
         }
 
         // create the new CastVoteRecord
-        CastVoteRecord newCvr =
-            new CastVoteRecord(Integer.toString(index), "no supplied ID", "no precinct", rankings);
+        CastVoteRecord newCvr = new CastVoteRecord(
+            Integer.toString(index),
+            "no supplied ID",
+            "no precinct",
+            "no batch ID",
+            rankings);
         castVoteRecords.add(newCvr);
       }
     } catch (IOException exception) {

src/main/java/network/brightspots/rcv/GuiConfigController.java

@@ -153,6 +153,8 @@ public class GuiConfigController implements Initializable {
   @FXML
   private TextField textFieldContestOffice;
   @FXML
+  private CheckBox checkBoxTabulateByBatch;
+  @FXML
   private CheckBox checkBoxTabulateByPrecinct;
   @FXML
   private CheckBox checkBoxGenerateCdfJson;
@@ -167,6 +169,8 @@ public class GuiConfigController implements Initializable {
   @FXML
   private TableColumn<CvrSource, String> tableColumnCvrIdCol;
   @FXML
+  private TableColumn<CvrSource, String> tableColumnCvrBatchCol;
+  @FXML
   private TableColumn<CvrSource, String> tableColumnCvrPrecinctCol;
   @FXML
   private TableColumn<CvrSource, String> tableColumnCvrOvervoteDelimiter;
@@ -199,6 +203,8 @@ public class GuiConfigController implements Initializable {
   @FXML
   private TextField textFieldCvrIdCol;
   @FXML
+  private TextField textFieldCvrBatchCol;
+  @FXML
   private TextField textFieldCvrPrecinctCol;
   @FXML
   private TextField textFieldCvrOvervoteDelimiter;
@@ -687,6 +693,7 @@ public void buttonAddCvrFileClicked() {
     String firstVoteColumnIndex = getTextOrEmptyString(textFieldCvrFirstVoteCol);
     String firstVoteRowIndex = getTextOrEmptyString(textFieldCvrFirstVoteRow);
     String idColumnIndex = getTextOrEmptyString(textFieldCvrIdCol);
+    String batchColumnIndex = getTextOrEmptyString(textFieldCvrBatchCol);
     String precinctColumnIndex = getTextOrEmptyString(textFieldCvrPrecinctCol);
     String overvoteDelimiter = getTextOrEmptyString(textFieldCvrOvervoteDelimiter);
     String provider = getProviderChoice(choiceCvrProvider).getInternalLabel();
@@ -703,6 +710,7 @@ public void buttonAddCvrFileClicked() {
               firstVoteColumnIndex,
               firstVoteRowIndex,
               idColumnIndex,
+              batchColumnIndex,
               precinctColumnIndex,
               overvoteDelimiter,
               provider,
@@ -732,6 +740,7 @@ private void clearBasicCvrValidationHighlighting() {
         textFieldCvrFirstVoteCol,
         textFieldCvrFirstVoteRow,
         textFieldCvrIdCol,
+        textFieldCvrBatchCol,
         textFieldCvrPrecinctCol,
         textFieldCvrOvervoteDelimiter,
         textFieldCvrOvervoteLabel,
@@ -767,6 +776,10 @@ private void highlightInputsFailingBasicCvrValidation(Set<ValidationError> valid
       addErrorStyling(textFieldCvrIdCol);
     }
 
+    if (validationErrors.contains(ValidationError.CVR_BATCH_COLUMN_INVALID)) {
+      addErrorStyling(textFieldCvrBatchCol);
+    }
+
     if (validationErrors.contains(ValidationError.CVR_PRECINCT_COLUMN_INVALID)) {
       addErrorStyling(textFieldCvrPrecinctCol);
     }
@@ -813,6 +826,8 @@ private void clearAndDisableCvrFilesTabFields() {
     textFieldCvrFirstVoteRow.setDisable(true);
     textFieldCvrIdCol.clear();
     textFieldCvrIdCol.setDisable(true);
+    textFieldCvrBatchCol.clear();
+    textFieldCvrBatchCol.setDisable(true);
     textFieldCvrPrecinctCol.clear();
     textFieldCvrPrecinctCol.setDisable(true);
     textFieldCvrOvervoteDelimiter.clear();
@@ -962,6 +977,7 @@ private void setDefaultValues() {
         ContestConfig.SUGGESTED_EXHAUST_ON_DUPLICATE_CANDIDATES);
 
     textFieldOutputDirectory.setText(ContestConfig.SUGGESTED_OUTPUT_DIRECTORY);
+    checkBoxTabulateByBatch.setSelected(ContestConfig.SUGGESTED_TABULATE_BY_BATCH);
     checkBoxTabulateByPrecinct.setSelected(ContestConfig.SUGGESTED_TABULATE_BY_PRECINCT);
     checkBoxGenerateCdfJson.setSelected(ContestConfig.SUGGESTED_GENERATE_CDF_JSON);
   }
@@ -994,6 +1010,7 @@ private void clearConfig() {
     checkBoxExhaustOnDuplicateCandidate.setSelected(false);
 
     textFieldOutputDirectory.clear();
+    checkBoxTabulateByBatch.setSelected(false);
     checkBoxTabulateByPrecinct.setSelected(false);
     checkBoxGenerateCdfJson.setSelected(false);
 
@@ -1201,6 +1218,7 @@ public LocalDate fromString(String string) {
               .setText(String.valueOf(ContestConfig.SUGGESTED_CVR_FIRST_VOTE_ROW));
           textFieldCvrIdCol.setDisable(false);
           textFieldCvrIdCol.setText(String.valueOf(ContestConfig.SUGGESTED_CVR_ID_COLUMN));
+          textFieldCvrBatchCol.setDisable(false);
           textFieldCvrPrecinctCol.setDisable(false);
           textFieldCvrPrecinctCol
               .setText(String.valueOf(ContestConfig.SUGGESTED_CVR_PRECINCT_COLUMN));
@@ -1250,6 +1268,7 @@ public LocalDate fromString(String string) {
         new EditableColumnString(tableColumnCvrFirstVoteCol, "firstVoteColumnIndex"),
         new EditableColumnString(tableColumnCvrFirstVoteRow, "firstVoteRowIndex"),
         new EditableColumnString(tableColumnCvrIdCol, "idColumnIndex"),
+        new EditableColumnString(tableColumnCvrBatchCol, "batchColumnIndex"),
         new EditableColumnString(tableColumnCvrPrecinctCol, "precinctColumnIndex"),
         new EditableColumnString(tableColumnCvrOvervoteDelimiter, "overvoteDelimiter"),
         new EditableColumnString(tableColumnCvrContestId, "contestId"),
@@ -1460,6 +1479,7 @@ private void loadConfig(ContestConfig config) throws ConfigVersionIsNewerThanApp
     }
     textFieldContestJurisdiction.setText(outputSettings.contestJurisdiction);
     textFieldContestOffice.setText(outputSettings.contestOffice);
+    checkBoxTabulateByBatch.setSelected(outputSettings.tabulateByBatch);
     checkBoxTabulateByPrecinct.setSelected(outputSettings.tabulateByPrecinct);
     checkBoxGenerateCdfJson.setSelected(outputSettings.generateCdfJson);
 
@@ -1549,6 +1569,7 @@ private RawContestConfig createRawContestConfig() {
         datePickerContestDate.getValue() != null ? datePickerContestDate.getValue().toString() : "";
     outputSettings.contestJurisdiction = getTextOrEmptyString(textFieldContestJurisdiction);
     outputSettings.contestOffice = getTextOrEmptyString(textFieldContestOffice);
+    outputSettings.tabulateByBatch = checkBoxTabulateByBatch.isSelected();
     outputSettings.tabulateByPrecinct = checkBoxTabulateByPrecinct.isSelected();
     outputSettings.generateCdfJson = checkBoxGenerateCdfJson.isSelected();
     config.outputSettings = outputSettings;
@@ -1558,6 +1579,8 @@ private RawContestConfig createRawContestConfig() {
       source.setFilePath(source.getFilePath() != null ? source.getFilePath().trim() : "");
       source.setIdColumnIndex(
           source.getIdColumnIndex() != null ? source.getIdColumnIndex().trim() : "");
+      source.setBatchColumnIndex(
+              source.getBatchColumnIndex() != null ? source.getBatchColumnIndex().trim() : "");
       source.setPrecinctColumnIndex(
           source.getPrecinctColumnIndex() != null ? source.getPrecinctColumnIndex().trim() : "");
       source.setOvervoteDelimiter(
@@ -1748,6 +1771,7 @@ protected void cleanUp() {
     protected void setUpTaskCompletionTriggers(Task<Boolean> task, String failureMessage) {
       task.setOnFailed(
           arg0 -> {
+            task.getException().printStackTrace();
             Logger.severe(failureMessage, task.getException());
             cleanUp();
           });