Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tabulate by Batch, and readiness for Tabulate By X #816

Merged
merged 10 commits into from
May 29, 2024
Merged

Tabulate by Batch, and readiness for Tabulate By X #816

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ab81e47
WIP
artoonie Mar 27, 2024
d97838b
complete initial implementation; stub of ES&S Batch ID implementation
artoonie Mar 29, 2024
67915cf
lint
artoonie Mar 30, 2024
bd027bd
split change into its own PR
artoonie Mar 31, 2024
3ffe6df
complete ES&S batch support
artoonie Apr 13, 2024
2faa642
add safety checks
artoonie Apr 13, 2024
66d5da7
Merge remote-tracking branch 'origin/develop' into feature/issue-807_…
artoonie Apr 30, 2024
982fe84
fix argument order
artoonie May 1, 2024
186df22
s/field/slice
artoonie May 26, 2024
6a014a5
address additional comments
artoonie May 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 11 additions & 8 deletions src/main/java/network/brightspots/rcv/CastVoteRecord.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,12 @@ class CastVoteRecord {
}

CastVoteRecord(
String computedId, String suppliedId, String precinct, List<Pair<Integer, String>> rankings) {
this(null, null, null, suppliedId, computedId, precinct, null, rankings);
String computedId,
String suppliedId,
String precinct,
String batchId,
List<Pair<Integer, String>> rankings) {
this(null, null, batchId, suppliedId, computedId, precinct, null, rankings);
}

String getContestId() {
Expand All @@ -101,12 +105,11 @@ String getTabulatorId() {
return tabulatorId;
}

String getBatchId() {
return batchId;
}

String getPrecinct() {
return precinct;
String getSlice(ContestConfig.TabulateBySlice slice) {
return switch (slice) {
case BATCH -> batchId;
case PRECINCT -> precinct;
};
}

String getPrecinctPortion() {
Expand Down
7 changes: 4 additions & 3 deletions src/main/java/network/brightspots/rcv/CommonDataFormatReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -315,8 +315,8 @@ void parseXml(List<CastVoteRecord> castVoteRecords) throws CvrParseException, IO

String computedCastVoteRecordId = String.format("%s(%d)", fileName, ++cvrIndex);
// create the new CastVoteRecord
CastVoteRecord newRecord =
new CastVoteRecord(computedCastVoteRecordId, cvr.UniqueId, precinctId, rankings);
CastVoteRecord newRecord = new CastVoteRecord(
computedCastVoteRecordId, cvr.UniqueId, precinctId, cvr.BatchSequenceId, rankings);
castVoteRecords.add(newRecord);

// provide some user feedback on the CVR count
Expand Down Expand Up @@ -524,10 +524,11 @@ void parseJson(List<CastVoteRecord> castVoteRecords) throws CvrParseException {
}

String ballotId = (String) cvr.get("BallotPrePrintedId");
String batchId = (String) cvr.get("BatchSequenceId");
String computedCastVoteRecordId = String.format("%s(%d)", fileName, ++cvrIndex);
// create the new CastVoteRecord
CastVoteRecord newRecord =
new CastVoteRecord(computedCastVoteRecordId, ballotId, precinctId, rankings);
new CastVoteRecord(computedCastVoteRecordId, ballotId, precinctId, batchId, rankings);
castVoteRecords.add(newRecord);
// provide some user feedback on the CVR count
if (castVoteRecords.size() % 50000 == 0) {
Expand Down
72 changes: 64 additions & 8 deletions src/main/java/network/brightspots/rcv/ContestConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import network.brightspots.rcv.RawContestConfig.Candidate;
import network.brightspots.rcv.RawContestConfig.CvrSource;
Expand All @@ -44,6 +45,7 @@ class ContestConfig {
// If any booleans are unspecified in config file, they should default to false no matter what
static final String AUTOMATED_TEST_VERSION = "TEST";
static final String SUGGESTED_OUTPUT_DIRECTORY = "output";
static final boolean SUGGESTED_TABULATE_BY_BATCH = false;
static final boolean SUGGESTED_TABULATE_BY_PRECINCT = false;
static final boolean SUGGESTED_GENERATE_CDF_JSON = false;
static final boolean SUGGESTED_CANDIDATE_EXCLUDED = false;
Expand Down Expand Up @@ -228,6 +230,14 @@ && stringAlreadyInUseElsewhereInSource(
source.getFilePath())) {
validationErrors.add(ValidationError.CVR_PRECINCT_COLUMN_UNEXPECTEDLY_DEFINED);
}

if (fieldIsDefinedButShouldNotBeForProvider(
source.getBatchColumnIndex(),
"batchColumnIndex",
provider,
source.getFilePath())) {
validationErrors.add(ValidationError.CVR_BATCH_COLUMN_UNEXPECTEDLY_DEFINED);
}
}

// See the config file documentation for an explanation of this regex
Expand Down Expand Up @@ -284,6 +294,14 @@ && stringAlreadyInUseElsewhereInSource(
validationErrors.add(ValidationError.CVR_PRECINCT_COLUMN_UNEXPECTEDLY_DEFINED);
}

if (fieldIsDefinedButShouldNotBeForProvider(
source.getBatchColumnIndex(),
"batchColumnIndex",
provider,
source.getFilePath())) {
validationErrors.add(ValidationError.CVR_BATCH_COLUMN_UNEXPECTEDLY_DEFINED);
}

if (fieldIsDefinedButShouldNotBeForProvider(
source.getOvervoteDelimiter(), "overvoteDelimiter", provider, source.getFilePath())) {
validationErrors.add(ValidationError.CVR_OVERVOTE_DELIMITER_UNEXPECTEDLY_DEFINED);
Expand Down Expand Up @@ -599,16 +617,26 @@ && getOvervoteRule() != Tabulator.OvervoteRule.ALWAYS_SKIP_TO_NEXT_RANK) {

if (getProvider(source) == Provider.CDF) {
// Perform CDF checks
if (isTabulateByPrecinctEnabled()) {
validationErrors.add(ValidationError.CVR_CDF_TABULATE_BY_PRECINCT_DISAGREEMENT);
Logger.severe("tabulateByPrecinct may not be used with CDF files.");
for (TabulateBySlice tabulateBySlice : enabledSlices()) {
validationErrors.add(ValidationError.CVR_CDF_TABULATE_BY_DISAGREEMENT);
Logger.severe("Tabulate-by-%s has not yet been implemented for CDF files.",
tabulateBySlice);
}
} else if (getProvider(source) == Provider.ESS) {
// Perform ES&S checks
if (isNullOrBlank(source.getPrecinctColumnIndex()) && isTabulateByPrecinctEnabled()) {
if (isNullOrBlank(source.getPrecinctColumnIndex())
&& isTabulateByEnabled(TabulateBySlice.PRECINCT)) {
validationErrors.add(ValidationError.CVR_TABULATE_BY_PRECINCT_REQUIRES_PRECINCT_COLUMN);
Logger.severe(
"precinctColumnIndex is required when tabulateByPrecinct is enabled: %s", cvrPath);
"precinctColumnIndex is required when tabulateByPrecinct is enabled: %s",
cvrPath);
}
if (isNullOrBlank(source.getBatchColumnIndex())
&& isTabulateByEnabled(TabulateBySlice.BATCH)) {
validationErrors.add(ValidationError.CVR_TABULATE_BY_PRECINCT_REQUIRES_BATCH_COLUMN);
Logger.severe(
"batchColumnIndex is required when tabulateByBatch is enabled: %s",
cvrPath);
}
if (isNullOrBlank(source.getOvervoteDelimiter())
&& getOvervoteRule() == OvervoteRule.EXHAUST_IF_MULTIPLE_CONTINUING) {
Expand Down Expand Up @@ -959,8 +987,17 @@ String getContestDate() {
return rawConfig.outputSettings.contestDate;
}

boolean isTabulateByPrecinctEnabled() {
return rawConfig.outputSettings.tabulateByPrecinct;
boolean isTabulateByEnabled(TabulateBySlice slice) {
return switch (slice) {
case PRECINCT -> rawConfig.outputSettings.tabulateByPrecinct;
case BATCH -> rawConfig.outputSettings.tabulateByBatch;
};
}

List<TabulateBySlice> enabledSlices() {
return Arrays.stream(TabulateBySlice.values())
.filter(this::isTabulateByEnabled)
.collect(Collectors.toList());
}

boolean isGenerateCdfJsonEnabled() {
Expand Down Expand Up @@ -1194,6 +1231,7 @@ enum ValidationError {
CVR_FIRST_VOTE_COLUMN_INVALID,
CVR_FIRST_VOTE_ROW_INVALID,
CVR_ID_COLUMN_INVALID,
CVR_BATCH_COLUMN_INVALID,
CVR_PRECINCT_COLUMN_INVALID,
CVR_OVERVOTE_DELIMITER_INVALID,
CVR_CDF_FILE_PATH_INVALID,
Expand All @@ -1202,7 +1240,8 @@ enum ValidationError {
CVR_DUPLICATE_FILE_PATHS,
CVR_FILE_PATH_INVALID,
CVR_OVERVOTE_LABEL_OVERVOTE_RULE_MISMATCH,
CVR_CDF_TABULATE_BY_PRECINCT_DISAGREEMENT,
CVR_CDF_TABULATE_BY_DISAGREEMENT,
CVR_TABULATE_BY_PRECINCT_REQUIRES_BATCH_COLUMN,
CVR_TABULATE_BY_PRECINCT_REQUIRES_PRECINCT_COLUMN,
CVR_OVERVOTE_DELIMITER_AND_LABEL_BOTH_SUPPLIED,
CVR_OVERVOTE_DELIMITER_MISSING,
Expand All @@ -1211,6 +1250,7 @@ enum ValidationError {
CVR_FIRST_VOTE_UNEXPECTEDLY_DEFINED,
CVR_FIRST_VOTE_ROW_UNEXPECTEDLY_DEFINED,
CVR_ID_COLUMN_UNEXPECTEDLY_DEFINED,
CVR_BATCH_COLUMN_UNEXPECTEDLY_DEFINED,
CVR_PRECINCT_COLUMN_UNEXPECTEDLY_DEFINED,
CVR_SKIPPED_RANK_LABEL_UNEXPECTEDLY_DEFINED,
CVR_CONTEST_ID_UNEXPECTEDLY_DEFINED,
Expand Down Expand Up @@ -1291,5 +1331,21 @@ public String getInternalLabel() {
}
}

enum TabulateBySlice {
PRECINCT("Precinct"),
BATCH("Batch");

private final String label;

TabulateBySlice(String label) {
this.label = label;
}

@Override
public String toString() {
return label;
}
}

static class UnrecognizedProviderException extends Exception {}
}
8 changes: 6 additions & 2 deletions src/main/java/network/brightspots/rcv/CsvCvrReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,12 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords)
}

// create the new CastVoteRecord
CastVoteRecord newCvr =
new CastVoteRecord(Integer.toString(index), "no supplied ID", "no precinct", rankings);
CastVoteRecord newCvr = new CastVoteRecord(
Integer.toString(index),
"no supplied ID",
"no precinct",
"no batch ID",
rankings);
castVoteRecords.add(newCvr);
}
} catch (IOException exception) {
Expand Down
24 changes: 24 additions & 0 deletions src/main/java/network/brightspots/rcv/GuiConfigController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,8 @@ public class GuiConfigController implements Initializable {
@FXML
private TextField textFieldContestOffice;
@FXML
private CheckBox checkBoxTabulateByBatch;
@FXML
private CheckBox checkBoxTabulateByPrecinct;
@FXML
private CheckBox checkBoxGenerateCdfJson;
Expand All @@ -158,6 +160,8 @@ public class GuiConfigController implements Initializable {
@FXML
private TableColumn<CvrSource, String> tableColumnCvrIdCol;
@FXML
private TableColumn<CvrSource, String> tableColumnCvrBatchCol;
@FXML
private TableColumn<CvrSource, String> tableColumnCvrPrecinctCol;
@FXML
private TableColumn<CvrSource, String> tableColumnCvrOvervoteDelimiter;
Expand Down Expand Up @@ -190,6 +194,8 @@ public class GuiConfigController implements Initializable {
@FXML
private TextField textFieldCvrIdCol;
@FXML
private TextField textFieldCvrBatchCol;
@FXML
private TextField textFieldCvrPrecinctCol;
@FXML
private TextField textFieldCvrOvervoteDelimiter;
Expand Down Expand Up @@ -615,6 +621,7 @@ public void buttonAddCvrFileClicked() {
String firstVoteColumnIndex = getTextOrEmptyString(textFieldCvrFirstVoteCol);
String firstVoteRowIndex = getTextOrEmptyString(textFieldCvrFirstVoteRow);
String idColumnIndex = getTextOrEmptyString(textFieldCvrIdCol);
String batchColumnIndex = getTextOrEmptyString(textFieldCvrBatchCol);
String precinctColumnIndex = getTextOrEmptyString(textFieldCvrPrecinctCol);
String overvoteDelimiter = getTextOrEmptyString(textFieldCvrOvervoteDelimiter);
String provider = getProviderChoice(choiceCvrProvider).getInternalLabel();
Expand All @@ -631,6 +638,7 @@ public void buttonAddCvrFileClicked() {
firstVoteColumnIndex,
firstVoteRowIndex,
idColumnIndex,
batchColumnIndex,
precinctColumnIndex,
overvoteDelimiter,
provider,
Expand Down Expand Up @@ -660,6 +668,7 @@ private void clearBasicCvrValidationHighlighting() {
textFieldCvrFirstVoteCol,
textFieldCvrFirstVoteRow,
textFieldCvrIdCol,
textFieldCvrBatchCol,
textFieldCvrPrecinctCol,
textFieldCvrOvervoteDelimiter,
textFieldCvrOvervoteLabel,
Expand Down Expand Up @@ -695,6 +704,10 @@ private void highlightInputsFailingBasicCvrValidation(Set<ValidationError> valid
addErrorStyling(textFieldCvrIdCol);
}

if (validationErrors.contains(ValidationError.CVR_BATCH_COLUMN_INVALID)) {
addErrorStyling(textFieldCvrBatchCol);
}

if (validationErrors.contains(ValidationError.CVR_PRECINCT_COLUMN_INVALID)) {
addErrorStyling(textFieldCvrPrecinctCol);
}
Expand Down Expand Up @@ -741,6 +754,8 @@ private void clearAndDisableCvrFilesTabFields() {
textFieldCvrFirstVoteRow.setDisable(true);
textFieldCvrIdCol.clear();
textFieldCvrIdCol.setDisable(true);
textFieldCvrBatchCol.clear();
textFieldCvrBatchCol.setDisable(true);
textFieldCvrPrecinctCol.clear();
textFieldCvrPrecinctCol.setDisable(true);
textFieldCvrOvervoteDelimiter.clear();
Expand Down Expand Up @@ -890,6 +905,7 @@ private void setDefaultValues() {
ContestConfig.SUGGESTED_EXHAUST_ON_DUPLICATE_CANDIDATES);

textFieldOutputDirectory.setText(ContestConfig.SUGGESTED_OUTPUT_DIRECTORY);
checkBoxTabulateByBatch.setSelected(ContestConfig.SUGGESTED_TABULATE_BY_BATCH);
checkBoxTabulateByPrecinct.setSelected(ContestConfig.SUGGESTED_TABULATE_BY_PRECINCT);
checkBoxGenerateCdfJson.setSelected(ContestConfig.SUGGESTED_GENERATE_CDF_JSON);
}
Expand Down Expand Up @@ -922,6 +938,7 @@ private void clearConfig() {
checkBoxExhaustOnDuplicateCandidate.setSelected(false);

textFieldOutputDirectory.clear();
checkBoxTabulateByBatch.setSelected(false);
checkBoxTabulateByPrecinct.setSelected(false);
checkBoxGenerateCdfJson.setSelected(false);

Expand Down Expand Up @@ -1125,6 +1142,7 @@ public LocalDate fromString(String string) {
.setText(String.valueOf(ContestConfig.SUGGESTED_CVR_FIRST_VOTE_ROW));
textFieldCvrIdCol.setDisable(false);
textFieldCvrIdCol.setText(String.valueOf(ContestConfig.SUGGESTED_CVR_ID_COLUMN));
textFieldCvrBatchCol.setDisable(false);
textFieldCvrPrecinctCol.setDisable(false);
textFieldCvrPrecinctCol
.setText(String.valueOf(ContestConfig.SUGGESTED_CVR_PRECINCT_COLUMN));
Expand Down Expand Up @@ -1174,6 +1192,7 @@ public LocalDate fromString(String string) {
new EditableColumnString(tableColumnCvrFirstVoteCol, "firstVoteColumnIndex"),
new EditableColumnString(tableColumnCvrFirstVoteRow, "firstVoteRowIndex"),
new EditableColumnString(tableColumnCvrIdCol, "idColumnIndex"),
new EditableColumnString(tableColumnCvrBatchCol, "batchColumnIndex"),
new EditableColumnString(tableColumnCvrPrecinctCol, "precinctColumnIndex"),
new EditableColumnString(tableColumnCvrOvervoteDelimiter, "overvoteDelimiter"),
new EditableColumnString(tableColumnCvrContestId, "contestId"),
Expand Down Expand Up @@ -1384,6 +1403,7 @@ private void loadConfig(ContestConfig config) throws ConfigVersionIsNewerThanApp
}
textFieldContestJurisdiction.setText(outputSettings.contestJurisdiction);
textFieldContestOffice.setText(outputSettings.contestOffice);
checkBoxTabulateByBatch.setSelected(outputSettings.tabulateByBatch);
checkBoxTabulateByPrecinct.setSelected(outputSettings.tabulateByPrecinct);
checkBoxGenerateCdfJson.setSelected(outputSettings.generateCdfJson);

Expand Down Expand Up @@ -1473,6 +1493,7 @@ private RawContestConfig createRawContestConfig() {
datePickerContestDate.getValue() != null ? datePickerContestDate.getValue().toString() : "";
outputSettings.contestJurisdiction = getTextOrEmptyString(textFieldContestJurisdiction);
outputSettings.contestOffice = getTextOrEmptyString(textFieldContestOffice);
outputSettings.tabulateByBatch = checkBoxTabulateByBatch.isSelected();
outputSettings.tabulateByPrecinct = checkBoxTabulateByPrecinct.isSelected();
outputSettings.generateCdfJson = checkBoxGenerateCdfJson.isSelected();
config.outputSettings = outputSettings;
Expand All @@ -1482,6 +1503,8 @@ private RawContestConfig createRawContestConfig() {
source.setFilePath(source.getFilePath() != null ? source.getFilePath().trim() : "");
source.setIdColumnIndex(
source.getIdColumnIndex() != null ? source.getIdColumnIndex().trim() : "");
source.setBatchColumnIndex(
source.getBatchColumnIndex() != null ? source.getBatchColumnIndex().trim() : "");
source.setPrecinctColumnIndex(
source.getPrecinctColumnIndex() != null ? source.getPrecinctColumnIndex().trim() : "");
source.setOvervoteDelimiter(
Expand Down Expand Up @@ -1665,6 +1688,7 @@ protected void cleanUp() {
protected void setUpTaskCompletionTriggers(Task<Void> task, String failureMessage) {
task.setOnFailed(
arg0 -> {
task.getException().printStackTrace();
Logger.severe(failureMessage, task.getException());
cleanUp();
});
Expand Down
Loading
Loading