Merge remote-tracking branch 'origin/develop' into feature/issue-679_…

…confirm-num-ballots

.github/workflows/release.yml

@@ -3,6 +3,8 @@ name: "Generate Releases"
 on:
   release:
     types: [ published ]
+  schedule:
+    - cron: '0 12 1,15 * *'  # On the 1st and 15th of the month at noon
 ## To test this workflow without creating a release, uncomment the following and add a branch name (making sure "push"
 ##  is at the same indent level as "release":
 #  push:
@@ -136,7 +138,7 @@ jobs:
 
       - name: "Upload binaries to release"
         uses: svenstaro/upload-release-action@v2
-        if: github.event_name == 'release'
+        if: github.event_name == 'release' || github.event_name == 'schedule'
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           file: build/${{ steps.basefn.outputs.FILEPATH }}*

src/main/java/network/brightspots/rcv/CastVoteRecord.java

@@ -66,23 +66,35 @@ class CastVoteRecord {
       String precinct,
       String precinctPortion,
       List<Pair<Integer, String>> rankings) {
+    this(contestId, tabulatorId, batchId, suppliedId, null, precinct, precinctPortion, rankings);
+  }
+
+  CastVoteRecord(
+          String contestId,
+          String tabulatorId,
+          String batchId,
+          String suppliedId,
+          String computedId,
+          String precinct,
+          String precinctPortion,
+          List<Pair<Integer, String>> rankings) {
     this.contestId = contestId;
     this.tabulatorId = tabulatorId;
     this.batchId = batchId;
-    this.computedId = null;
     this.suppliedId = suppliedId;
+    this.computedId = computedId;
     this.precinct = precinct;
     this.precinctPortion = precinctPortion;
     this.candidateRankings = new CandidateRankingsList(rankings);
   }
 
   CastVoteRecord(
-      String computedId, String suppliedId, String precinct, List<Pair<Integer, String>> rankings) {
-    this.computedId = computedId;
-    this.suppliedId = suppliedId;
-    this.precinct = precinct;
-    this.precinctPortion = null;
-    this.candidateRankings = new CandidateRankingsList(rankings);
+      String computedId,
+      String suppliedId,
+      String precinct,
+      String batchId,
+      List<Pair<Integer, String>> rankings) {
+    this(null, null, batchId, suppliedId, computedId, precinct, null, rankings);
   }
 
   String getContestId() {
@@ -93,12 +105,11 @@ String getTabulatorId() {
     return tabulatorId;
   }
 
-  String getBatchId() {
-    return batchId;
-  }
-
-  String getPrecinct() {
-    return precinct;
+  String getSlice(ContestConfig.TabulateBySlice slice) {
+    return switch (slice) {
+      case BATCH -> batchId;
+      case PRECINCT -> precinct;
+    };
   }
 
   String getPrecinctPortion() {
@@ -115,10 +126,10 @@ void logRoundOutcome(
 
     StringBuilder logStringBuilder = new StringBuilder();
     logStringBuilder.append("[Round] ").append(round).append(" [CVR] ");
-    if (!isNullOrBlank(suppliedId)) {
-      logStringBuilder.append(suppliedId);
-    } else {
+    if (!isNullOrBlank(computedId)) {
       logStringBuilder.append(computedId);
+    } else {
+      logStringBuilder.append(suppliedId);
     }
     if (outcomeType == VoteOutcomeType.IGNORED) {
       logStringBuilder.append(" [was ignored] ");

src/main/java/network/brightspots/rcv/CommonDataFormatReader.java

@@ -315,8 +315,8 @@ void parseXml(List<CastVoteRecord> castVoteRecords) throws CvrParseException, IO
 
         String computedCastVoteRecordId = String.format("%s(%d)", fileName, ++cvrIndex);
         // create the new CastVoteRecord
-        CastVoteRecord newRecord =
-            new CastVoteRecord(computedCastVoteRecordId, cvr.UniqueId, precinctId, rankings);
+        CastVoteRecord newRecord = new CastVoteRecord(
+            computedCastVoteRecordId, cvr.UniqueId, precinctId, cvr.BatchSequenceId, rankings);
         castVoteRecords.add(newRecord);
 
         // provide some user feedback on the CVR count
@@ -524,10 +524,11 @@ void parseJson(List<CastVoteRecord> castVoteRecords) throws CvrParseException {
       }
 
       String ballotId = (String) cvr.get("BallotPrePrintedId");
+      String batchId = (String) cvr.get("BatchSequenceId");
       String computedCastVoteRecordId = String.format("%s(%d)", fileName, ++cvrIndex);
       // create the new CastVoteRecord
       CastVoteRecord newRecord =
-          new CastVoteRecord(computedCastVoteRecordId, ballotId, precinctId, rankings);
+          new CastVoteRecord(computedCastVoteRecordId, ballotId, precinctId, batchId, rankings);
       castVoteRecords.add(newRecord);
       // provide some user feedback on the CVR count
       if (castVoteRecords.size() % 50000 == 0) {

src/main/java/network/brightspots/rcv/ContestConfig.java

@@ -32,6 +32,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import network.brightspots.rcv.RawContestConfig.Candidate;
 import network.brightspots.rcv.RawContestConfig.CvrSource;
@@ -44,6 +45,7 @@ class ContestConfig {
   // If any booleans are unspecified in config file, they should default to false no matter what
   static final String AUTOMATED_TEST_VERSION = "TEST";
   static final String SUGGESTED_OUTPUT_DIRECTORY = "output";
+  static final boolean SUGGESTED_TABULATE_BY_BATCH = false;
   static final boolean SUGGESTED_TABULATE_BY_PRECINCT = false;
   static final boolean SUGGESTED_GENERATE_CDF_JSON = false;
   static final boolean SUGGESTED_CANDIDATE_EXCLUDED = false;
@@ -228,6 +230,14 @@ && stringAlreadyInUseElsewhereInSource(
               source.getFilePath())) {
             validationErrors.add(ValidationError.CVR_PRECINCT_COLUMN_UNEXPECTEDLY_DEFINED);
           }
+
+          if (fieldIsDefinedButShouldNotBeForProvider(
+                  source.getBatchColumnIndex(),
+                  "batchColumnIndex",
+                  provider,
+                  source.getFilePath())) {
+            validationErrors.add(ValidationError.CVR_BATCH_COLUMN_UNEXPECTEDLY_DEFINED);
+          }
         }
 
         // See the config file documentation for an explanation of this regex
@@ -284,6 +294,14 @@ && stringAlreadyInUseElsewhereInSource(
           validationErrors.add(ValidationError.CVR_PRECINCT_COLUMN_UNEXPECTEDLY_DEFINED);
         }
 
+        if (fieldIsDefinedButShouldNotBeForProvider(
+                source.getBatchColumnIndex(),
+                "batchColumnIndex",
+                provider,
+                source.getFilePath())) {
+          validationErrors.add(ValidationError.CVR_BATCH_COLUMN_UNEXPECTEDLY_DEFINED);
+        }
+
         if (fieldIsDefinedButShouldNotBeForProvider(
             source.getOvervoteDelimiter(), "overvoteDelimiter", provider, source.getFilePath())) {
           validationErrors.add(ValidationError.CVR_OVERVOTE_DELIMITER_UNEXPECTEDLY_DEFINED);
@@ -599,16 +617,26 @@ && getOvervoteRule() != Tabulator.OvervoteRule.ALWAYS_SKIP_TO_NEXT_RANK) {
 
         if (getProvider(source) == Provider.CDF) {
           // Perform CDF checks
-          if (isTabulateByPrecinctEnabled()) {
-            validationErrors.add(ValidationError.CVR_CDF_TABULATE_BY_PRECINCT_DISAGREEMENT);
-            Logger.severe("tabulateByPrecinct may not be used with CDF files.");
+          for (TabulateBySlice tabulateBySlice : enabledSlices()) {
+            validationErrors.add(ValidationError.CVR_CDF_TABULATE_BY_DISAGREEMENT);
+            Logger.severe("Tabulate-by-%s has not yet been implemented for CDF files.",
+                    tabulateBySlice);
           }
         } else if (getProvider(source) == Provider.ESS) {
           // Perform ES&S checks
-          if (isNullOrBlank(source.getPrecinctColumnIndex()) && isTabulateByPrecinctEnabled()) {
+          if (isNullOrBlank(source.getPrecinctColumnIndex())
+              && isTabulateByEnabled(TabulateBySlice.PRECINCT)) {
             validationErrors.add(ValidationError.CVR_TABULATE_BY_PRECINCT_REQUIRES_PRECINCT_COLUMN);
             Logger.severe(
-                "precinctColumnIndex is required when tabulateByPrecinct is enabled: %s", cvrPath);
+                "precinctColumnIndex is required when tabulateByPrecinct is enabled: %s",
+                cvrPath);
+          }
+          if (isNullOrBlank(source.getBatchColumnIndex())
+              && isTabulateByEnabled(TabulateBySlice.BATCH)) {
+            validationErrors.add(ValidationError.CVR_TABULATE_BY_PRECINCT_REQUIRES_BATCH_COLUMN);
+            Logger.severe(
+                    "batchColumnIndex is required when tabulateByBatch is enabled: %s",
+                    cvrPath);
           }
           if (isNullOrBlank(source.getOvervoteDelimiter())
               && getOvervoteRule() == OvervoteRule.EXHAUST_IF_MULTIPLE_CONTINUING) {
@@ -959,8 +987,17 @@ String getContestDate() {
     return rawConfig.outputSettings.contestDate;
   }
 
-  boolean isTabulateByPrecinctEnabled() {
-    return rawConfig.outputSettings.tabulateByPrecinct;
+  boolean isTabulateByEnabled(TabulateBySlice slice) {
+    return switch (slice) {
+      case PRECINCT -> rawConfig.outputSettings.tabulateByPrecinct;
+      case BATCH -> rawConfig.outputSettings.tabulateByBatch;
+    };
+  }
+
+  List<TabulateBySlice> enabledSlices() {
+    return Arrays.stream(TabulateBySlice.values())
+            .filter(this::isTabulateByEnabled)
+            .collect(Collectors.toList());
   }
 
   boolean isGenerateCdfJsonEnabled() {
@@ -1194,6 +1231,7 @@ enum ValidationError {
     CVR_FIRST_VOTE_COLUMN_INVALID,
     CVR_FIRST_VOTE_ROW_INVALID,
     CVR_ID_COLUMN_INVALID,
+    CVR_BATCH_COLUMN_INVALID,
     CVR_PRECINCT_COLUMN_INVALID,
     CVR_OVERVOTE_DELIMITER_INVALID,
     CVR_CDF_FILE_PATH_INVALID,
@@ -1202,7 +1240,8 @@ enum ValidationError {
     CVR_DUPLICATE_FILE_PATHS,
     CVR_FILE_PATH_INVALID,
     CVR_OVERVOTE_LABEL_OVERVOTE_RULE_MISMATCH,
-    CVR_CDF_TABULATE_BY_PRECINCT_DISAGREEMENT,
+    CVR_CDF_TABULATE_BY_DISAGREEMENT,
+    CVR_TABULATE_BY_PRECINCT_REQUIRES_BATCH_COLUMN,
     CVR_TABULATE_BY_PRECINCT_REQUIRES_PRECINCT_COLUMN,
     CVR_OVERVOTE_DELIMITER_AND_LABEL_BOTH_SUPPLIED,
     CVR_OVERVOTE_DELIMITER_MISSING,
@@ -1211,6 +1250,7 @@ enum ValidationError {
     CVR_FIRST_VOTE_UNEXPECTEDLY_DEFINED,
     CVR_FIRST_VOTE_ROW_UNEXPECTEDLY_DEFINED,
     CVR_ID_COLUMN_UNEXPECTEDLY_DEFINED,
+    CVR_BATCH_COLUMN_UNEXPECTEDLY_DEFINED,
     CVR_PRECINCT_COLUMN_UNEXPECTEDLY_DEFINED,
     CVR_SKIPPED_RANK_LABEL_UNEXPECTEDLY_DEFINED,
     CVR_CONTEST_ID_UNEXPECTEDLY_DEFINED,
@@ -1291,5 +1331,25 @@ public String getInternalLabel() {
     }
   }
 
+  enum TabulateBySlice {
+    PRECINCT("Precinct"),
+    BATCH("Batch");
+
+    private final String label;
+
+    TabulateBySlice(String label) {
+      this.label = label;
+    }
+
+    @Override
+    public String toString() {
+      return label;
+    }
+
+    public String toLowerString() {
+      return label.toLowerCase();
+    }
+  }
+
   static class UnrecognizedProviderException extends Exception {}
 }

src/main/java/network/brightspots/rcv/CsvCvrReader.java

@@ -79,7 +79,9 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords)
 
       parser.stream().skip(firstVoteRowIndex);
 
+      int index = 0;
       for (CSVRecord csvRecord : parser) {
+        index++;
         ArrayList<Pair<Integer, String>> rankings = new ArrayList<>();
         for (int col = firstVoteColumnIndex; col < csvRecord.size(); col++) {
           String rankAsString = csvRecord.get(col);
@@ -105,8 +107,12 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords)
         }
 
         // create the new CastVoteRecord
-        CastVoteRecord newCvr =
-            new CastVoteRecord(source.getContestId(), "no supplied ID", "no precinct", rankings);
+        CastVoteRecord newCvr = new CastVoteRecord(
+            Integer.toString(index),
+            "no supplied ID",
+            "no precinct",
+            "no batch ID",
+            rankings);
         castVoteRecords.add(newCvr);
       }
     } catch (IOException exception) {

src/main/java/network/brightspots/rcv/DominionCvrReader.java

@@ -30,6 +30,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import javafx.util.Pair;
 import network.brightspots.rcv.CastVoteRecord.CvrParseException;
 
@@ -271,6 +273,10 @@ private int parseCvrFile(
       String batchId = session.get("BatchId").toString();
       Integer recordId = (Integer) session.get("RecordId");
       String suppliedId = recordId.toString();
+      String computedId = Stream.of(tabulatorId, batchId, Integer.toString(recordId))
+              .filter(s -> s != null && !s.isEmpty())
+              .collect(Collectors.joining("|"));
+
       // filter out records which are not current and replace them with adjudicated ones
       HashMap adjudicatedData = (HashMap) session.get("Original");
       boolean isCurrent = (boolean) adjudicatedData.get("IsCurrent");
@@ -349,8 +355,8 @@ private int parseCvrFile(
           }
           // create the new cvr
           CastVoteRecord newCvr =
-              new CastVoteRecord(
-                  contestId, tabulatorId, batchId, suppliedId, precinct, precinctPortion, rankings);
+              new CastVoteRecord(contestId, tabulatorId, batchId, suppliedId,
+                      computedId, precinct, precinctPortion, rankings);
           castVoteRecords.add(newCvr);
         }
       }