Skip to content

Commit

Permalink
Tabulate by Batch, and readiness for Tabulate By X (#816)
Browse files Browse the repository at this point in the history 
* WIP

* complete initial implementation; stub of ES&S Batch ID implementation

* lint

* split change into its own PR

* complete ES&S batch support

* add safety checks

* fix argument order

* s/field/slice

* address additional comments

---------

Co-authored-by: Armin Samii <[email protected]>
  • Loading branch information
@artoonie
artoonie and artoonie authored May 29, 2024
1 parent 6c2a362 commit 9e95760
Show file tree
Hide file tree
Showing 18 changed files with 21,349 additions and 173 deletions.
19 changes: 11 additions & 8 deletions src/main/java/network/brightspots/rcv/CastVoteRecord.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,12 @@ class CastVoteRecord {
}

CastVoteRecord(
String computedId, String suppliedId, String precinct, List<Pair<Integer, String>> rankings) {
this(null, null, null, suppliedId, computedId, precinct, null, rankings);
String computedId,
String suppliedId,
String precinct,
String batchId,
List<Pair<Integer, String>> rankings) {
this(null, null, batchId, suppliedId, computedId, precinct, null, rankings);
}

String getContestId() {
Expand All @@ -101,12 +105,11 @@ String getTabulatorId() {
return tabulatorId;
}

String getBatchId() {
return batchId;
}

String getPrecinct() {
return precinct;
String getSlice(ContestConfig.TabulateBySlice slice) {
return switch (slice) {
case BATCH -> batchId;
case PRECINCT -> precinct;
};
}

String getPrecinctPortion() {
Expand Down
7 changes: 4 additions & 3 deletions src/main/java/network/brightspots/rcv/CommonDataFormatReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,8 +315,8 @@ void parseXml(List<CastVoteRecord> castVoteRecords) throws CvrParseException, IO

String computedCastVoteRecordId = String.format("%s(%d)", fileName, ++cvrIndex);
// create the new CastVoteRecord
CastVoteRecord newRecord =
new CastVoteRecord(computedCastVoteRecordId, cvr.UniqueId, precinctId, rankings);
CastVoteRecord newRecord = new CastVoteRecord(
computedCastVoteRecordId, cvr.UniqueId, precinctId, cvr.BatchSequenceId, rankings);
castVoteRecords.add(newRecord);

// provide some user feedback on the CVR count
Expand Down Expand Up @@ -524,10 +524,11 @@ void parseJson(List<CastVoteRecord> castVoteRecords) throws CvrParseException {
}

String ballotId = (String) cvr.get("BallotPrePrintedId");
String batchId = (String) cvr.get("BatchSequenceId");
String computedCastVoteRecordId = String.format("%s(%d)", fileName, ++cvrIndex);
// create the new CastVoteRecord
CastVoteRecord newRecord =
new CastVoteRecord(computedCastVoteRecordId, ballotId, precinctId, rankings);
new CastVoteRecord(computedCastVoteRecordId, ballotId, precinctId, batchId, rankings);
castVoteRecords.add(newRecord);
// provide some user feedback on the CVR count
if (castVoteRecords.size() % 50000 == 0) {
Expand Down
72 changes: 64 additions & 8 deletions src/main/java/network/brightspots/rcv/ContestConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import network.brightspots.rcv.RawContestConfig.Candidate;
import network.brightspots.rcv.RawContestConfig.CvrSource;
Expand All @@ -44,6 +45,7 @@ class ContestConfig {
// If any booleans are unspecified in config file, they should default to false no matter what
static final String AUTOMATED_TEST_VERSION = "TEST";
static final String SUGGESTED_OUTPUT_DIRECTORY = "output";
static final boolean SUGGESTED_TABULATE_BY_BATCH = false;
static final boolean SUGGESTED_TABULATE_BY_PRECINCT = false;
static final boolean SUGGESTED_GENERATE_CDF_JSON = false;
static final boolean SUGGESTED_CANDIDATE_EXCLUDED = false;
Expand Down Expand Up @@ -228,6 +230,14 @@ && stringAlreadyInUseElsewhereInSource(
source.getFilePath())) {
validationErrors.add(ValidationError.CVR_PRECINCT_COLUMN_UNEXPECTEDLY_DEFINED);
}

if (fieldIsDefinedButShouldNotBeForProvider(
source.getBatchColumnIndex(),
"batchColumnIndex",
provider,
source.getFilePath())) {
validationErrors.add(ValidationError.CVR_BATCH_COLUMN_UNEXPECTEDLY_DEFINED);
}
}

// See the config file documentation for an explanation of this regex
Expand Down Expand Up @@ -284,6 +294,14 @@ && stringAlreadyInUseElsewhereInSource(
validationErrors.add(ValidationError.CVR_PRECINCT_COLUMN_UNEXPECTEDLY_DEFINED);
}

if (fieldIsDefinedButShouldNotBeForProvider(
source.getBatchColumnIndex(),
"batchColumnIndex",
provider,
source.getFilePath())) {
validationErrors.add(ValidationError.CVR_BATCH_COLUMN_UNEXPECTEDLY_DEFINED);
}

if (fieldIsDefinedButShouldNotBeForProvider(
source.getOvervoteDelimiter(), "overvoteDelimiter", provider, source.getFilePath())) {
validationErrors.add(ValidationError.CVR_OVERVOTE_DELIMITER_UNEXPECTEDLY_DEFINED);
Expand Down Expand Up @@ -599,16 +617,26 @@ && getOvervoteRule() != Tabulator.OvervoteRule.ALWAYS_SKIP_TO_NEXT_RANK) {

if (getProvider(source) == Provider.CDF) {
// Perform CDF checks
if (isTabulateByPrecinctEnabled()) {
validationErrors.add(ValidationError.CVR_CDF_TABULATE_BY_PRECINCT_DISAGREEMENT);
Logger.severe("tabulateByPrecinct may not be used with CDF files.");
for (TabulateBySlice tabulateBySlice : enabledSlices()) {
validationErrors.add(ValidationError.CVR_CDF_TABULATE_BY_DISAGREEMENT);
Logger.severe("Tabulate-by-%s has not yet been implemented for CDF files.",
tabulateBySlice);
}
} else if (getProvider(source) == Provider.ESS) {
// Perform ES&S checks
if (isNullOrBlank(source.getPrecinctColumnIndex()) && isTabulateByPrecinctEnabled()) {
if (isNullOrBlank(source.getPrecinctColumnIndex())
&& isTabulateByEnabled(TabulateBySlice.PRECINCT)) {
validationErrors.add(ValidationError.CVR_TABULATE_BY_PRECINCT_REQUIRES_PRECINCT_COLUMN);
Logger.severe(
"precinctColumnIndex is required when tabulateByPrecinct is enabled: %s", cvrPath);
"precinctColumnIndex is required when tabulateByPrecinct is enabled: %s",
cvrPath);
}
if (isNullOrBlank(source.getBatchColumnIndex())
&& isTabulateByEnabled(TabulateBySlice.BATCH)) {
validationErrors.add(ValidationError.CVR_TABULATE_BY_PRECINCT_REQUIRES_BATCH_COLUMN);
Logger.severe(
"batchColumnIndex is required when tabulateByBatch is enabled: %s",
cvrPath);
}
if (isNullOrBlank(source.getOvervoteDelimiter())
&& getOvervoteRule() == OvervoteRule.EXHAUST_IF_MULTIPLE_CONTINUING) {
Expand Down Expand Up @@ -959,8 +987,17 @@ String getContestDate() {
return rawConfig.outputSettings.contestDate;
}

boolean isTabulateByPrecinctEnabled() {
return rawConfig.outputSettings.tabulateByPrecinct;
boolean isTabulateByEnabled(TabulateBySlice slice) {
return switch (slice) {
case PRECINCT -> rawConfig.outputSettings.tabulateByPrecinct;
case BATCH -> rawConfig.outputSettings.tabulateByBatch;
};
}

List<TabulateBySlice> enabledSlices() {
return Arrays.stream(TabulateBySlice.values())
.filter(this::isTabulateByEnabled)
.collect(Collectors.toList());
}

boolean isGenerateCdfJsonEnabled() {
Expand Down Expand Up @@ -1194,6 +1231,7 @@ enum ValidationError {
CVR_FIRST_VOTE_COLUMN_INVALID,
CVR_FIRST_VOTE_ROW_INVALID,
CVR_ID_COLUMN_INVALID,
CVR_BATCH_COLUMN_INVALID,
CVR_PRECINCT_COLUMN_INVALID,
CVR_OVERVOTE_DELIMITER_INVALID,
CVR_CDF_FILE_PATH_INVALID,
Expand All @@ -1202,7 +1240,8 @@ enum ValidationError {
CVR_DUPLICATE_FILE_PATHS,
CVR_FILE_PATH_INVALID,
CVR_OVERVOTE_LABEL_OVERVOTE_RULE_MISMATCH,
CVR_CDF_TABULATE_BY_PRECINCT_DISAGREEMENT,
CVR_CDF_TABULATE_BY_DISAGREEMENT,
CVR_TABULATE_BY_PRECINCT_REQUIRES_BATCH_COLUMN,
CVR_TABULATE_BY_PRECINCT_REQUIRES_PRECINCT_COLUMN,
CVR_OVERVOTE_DELIMITER_AND_LABEL_BOTH_SUPPLIED,
CVR_OVERVOTE_DELIMITER_MISSING,
Expand All @@ -1211,6 +1250,7 @@ enum ValidationError {
CVR_FIRST_VOTE_UNEXPECTEDLY_DEFINED,
CVR_FIRST_VOTE_ROW_UNEXPECTEDLY_DEFINED,
CVR_ID_COLUMN_UNEXPECTEDLY_DEFINED,
CVR_BATCH_COLUMN_UNEXPECTEDLY_DEFINED,
CVR_PRECINCT_COLUMN_UNEXPECTEDLY_DEFINED,
CVR_SKIPPED_RANK_LABEL_UNEXPECTEDLY_DEFINED,
CVR_CONTEST_ID_UNEXPECTEDLY_DEFINED,
Expand Down Expand Up @@ -1291,5 +1331,21 @@ public String getInternalLabel() {
}
}

enum TabulateBySlice {
PRECINCT("Precinct"),
BATCH("Batch");

private final String label;

TabulateBySlice(String label) {
this.label = label;
}

@Override
public String toString() {
return label;
}
}

static class UnrecognizedProviderException extends Exception {}
}
8 changes: 6 additions & 2 deletions src/main/java/network/brightspots/rcv/CsvCvrReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,12 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords)
}

// create the new CastVoteRecord
CastVoteRecord newCvr =
new CastVoteRecord(Integer.toString(index), "no supplied ID", "no precinct", rankings);
CastVoteRecord newCvr = new CastVoteRecord(
Integer.toString(index),
"no supplied ID",
"no precinct",
"no batch ID",
rankings);
castVoteRecords.add(newCvr);
}
} catch (IOException exception) {
Expand Down
24 changes: 24 additions & 0 deletions src/main/java/network/brightspots/rcv/GuiConfigController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,8 @@ public class GuiConfigController implements Initializable {
@FXML
private TextField textFieldContestOffice;
@FXML
private CheckBox checkBoxTabulateByBatch;
@FXML
private CheckBox checkBoxTabulateByPrecinct;
@FXML
private CheckBox checkBoxGenerateCdfJson;
Expand All @@ -158,6 +160,8 @@ public class GuiConfigController implements Initializable {
@FXML
private TableColumn<CvrSource, String> tableColumnCvrIdCol;
@FXML
private TableColumn<CvrSource, String> tableColumnCvrBatchCol;
@FXML
private TableColumn<CvrSource, String> tableColumnCvrPrecinctCol;
@FXML
private TableColumn<CvrSource, String> tableColumnCvrOvervoteDelimiter;
Expand Down Expand Up @@ -190,6 +194,8 @@ public class GuiConfigController implements Initializable {
@FXML
private TextField textFieldCvrIdCol;
@FXML
private TextField textFieldCvrBatchCol;
@FXML
private TextField textFieldCvrPrecinctCol;
@FXML
private TextField textFieldCvrOvervoteDelimiter;
Expand Down Expand Up @@ -615,6 +621,7 @@ public void buttonAddCvrFileClicked() {
String firstVoteColumnIndex = getTextOrEmptyString(textFieldCvrFirstVoteCol);
String firstVoteRowIndex = getTextOrEmptyString(textFieldCvrFirstVoteRow);
String idColumnIndex = getTextOrEmptyString(textFieldCvrIdCol);
String batchColumnIndex = getTextOrEmptyString(textFieldCvrBatchCol);
String precinctColumnIndex = getTextOrEmptyString(textFieldCvrPrecinctCol);
String overvoteDelimiter = getTextOrEmptyString(textFieldCvrOvervoteDelimiter);
String provider = getProviderChoice(choiceCvrProvider).getInternalLabel();
Expand All @@ -631,6 +638,7 @@ public void buttonAddCvrFileClicked() {
firstVoteColumnIndex,
firstVoteRowIndex,
idColumnIndex,
batchColumnIndex,
precinctColumnIndex,
overvoteDelimiter,
provider,
Expand Down Expand Up @@ -660,6 +668,7 @@ private void clearBasicCvrValidationHighlighting() {
textFieldCvrFirstVoteCol,
textFieldCvrFirstVoteRow,
textFieldCvrIdCol,
textFieldCvrBatchCol,
textFieldCvrPrecinctCol,
textFieldCvrOvervoteDelimiter,
textFieldCvrOvervoteLabel,
Expand Down Expand Up @@ -695,6 +704,10 @@ private void highlightInputsFailingBasicCvrValidation(Set<ValidationError> valid
addErrorStyling(textFieldCvrIdCol);
}

if (validationErrors.contains(ValidationError.CVR_BATCH_COLUMN_INVALID)) {
addErrorStyling(textFieldCvrBatchCol);
}

if (validationErrors.contains(ValidationError.CVR_PRECINCT_COLUMN_INVALID)) {
addErrorStyling(textFieldCvrPrecinctCol);
}
Expand Down Expand Up @@ -741,6 +754,8 @@ private void clearAndDisableCvrFilesTabFields() {
textFieldCvrFirstVoteRow.setDisable(true);
textFieldCvrIdCol.clear();
textFieldCvrIdCol.setDisable(true);
textFieldCvrBatchCol.clear();
textFieldCvrBatchCol.setDisable(true);
textFieldCvrPrecinctCol.clear();
textFieldCvrPrecinctCol.setDisable(true);
textFieldCvrOvervoteDelimiter.clear();
Expand Down Expand Up @@ -890,6 +905,7 @@ private void setDefaultValues() {
ContestConfig.SUGGESTED_EXHAUST_ON_DUPLICATE_CANDIDATES);

textFieldOutputDirectory.setText(ContestConfig.SUGGESTED_OUTPUT_DIRECTORY);
checkBoxTabulateByBatch.setSelected(ContestConfig.SUGGESTED_TABULATE_BY_BATCH);
checkBoxTabulateByPrecinct.setSelected(ContestConfig.SUGGESTED_TABULATE_BY_PRECINCT);
checkBoxGenerateCdfJson.setSelected(ContestConfig.SUGGESTED_GENERATE_CDF_JSON);
}
Expand Down Expand Up @@ -922,6 +938,7 @@ private void clearConfig() {
checkBoxExhaustOnDuplicateCandidate.setSelected(false);

textFieldOutputDirectory.clear();
checkBoxTabulateByBatch.setSelected(false);
checkBoxTabulateByPrecinct.setSelected(false);
checkBoxGenerateCdfJson.setSelected(false);

Expand Down Expand Up @@ -1125,6 +1142,7 @@ public LocalDate fromString(String string) {
.setText(String.valueOf(ContestConfig.SUGGESTED_CVR_FIRST_VOTE_ROW));
textFieldCvrIdCol.setDisable(false);
textFieldCvrIdCol.setText(String.valueOf(ContestConfig.SUGGESTED_CVR_ID_COLUMN));
textFieldCvrBatchCol.setDisable(false);
textFieldCvrPrecinctCol.setDisable(false);
textFieldCvrPrecinctCol
.setText(String.valueOf(ContestConfig.SUGGESTED_CVR_PRECINCT_COLUMN));
Expand Down Expand Up @@ -1174,6 +1192,7 @@ public LocalDate fromString(String string) {
new EditableColumnString(tableColumnCvrFirstVoteCol, "firstVoteColumnIndex"),
new EditableColumnString(tableColumnCvrFirstVoteRow, "firstVoteRowIndex"),
new EditableColumnString(tableColumnCvrIdCol, "idColumnIndex"),
new EditableColumnString(tableColumnCvrBatchCol, "batchColumnIndex"),
new EditableColumnString(tableColumnCvrPrecinctCol, "precinctColumnIndex"),
new EditableColumnString(tableColumnCvrOvervoteDelimiter, "overvoteDelimiter"),
new EditableColumnString(tableColumnCvrContestId, "contestId"),
Expand Down Expand Up @@ -1384,6 +1403,7 @@ private void loadConfig(ContestConfig config) throws ConfigVersionIsNewerThanApp
}
textFieldContestJurisdiction.setText(outputSettings.contestJurisdiction);
textFieldContestOffice.setText(outputSettings.contestOffice);
checkBoxTabulateByBatch.setSelected(outputSettings.tabulateByBatch);
checkBoxTabulateByPrecinct.setSelected(outputSettings.tabulateByPrecinct);
checkBoxGenerateCdfJson.setSelected(outputSettings.generateCdfJson);

Expand Down Expand Up @@ -1473,6 +1493,7 @@ private RawContestConfig createRawContestConfig() {
datePickerContestDate.getValue() != null ? datePickerContestDate.getValue().toString() : "";
outputSettings.contestJurisdiction = getTextOrEmptyString(textFieldContestJurisdiction);
outputSettings.contestOffice = getTextOrEmptyString(textFieldContestOffice);
outputSettings.tabulateByBatch = checkBoxTabulateByBatch.isSelected();
outputSettings.tabulateByPrecinct = checkBoxTabulateByPrecinct.isSelected();
outputSettings.generateCdfJson = checkBoxGenerateCdfJson.isSelected();
config.outputSettings = outputSettings;
Expand All @@ -1482,6 +1503,8 @@ private RawContestConfig createRawContestConfig() {
source.setFilePath(source.getFilePath() != null ? source.getFilePath().trim() : "");
source.setIdColumnIndex(
source.getIdColumnIndex() != null ? source.getIdColumnIndex().trim() : "");
source.setBatchColumnIndex(
source.getBatchColumnIndex() != null ? source.getBatchColumnIndex().trim() : "");
source.setPrecinctColumnIndex(
source.getPrecinctColumnIndex() != null ? source.getPrecinctColumnIndex().trim() : "");
source.setOvervoteDelimiter(
Expand Down Expand Up @@ -1665,6 +1688,7 @@ protected void cleanUp() {
protected void setUpTaskCompletionTriggers(Task<Void> task, String failureMessage) {
task.setOnFailed(
arg0 -> {
task.getException().printStackTrace();
Logger.severe(failureMessage, task.getException());
cleanUp();
});
Expand Down
Loading

0 comments on commit 9e95760

Please sign in to comment.