Skip to content

Commit

Permalink
output folder DNE fix and audit log addition (#847)
Browse files Browse the repository at this point in the history 
* output folder DNE fix and audit log addition

* fix broken test

* fix compilation error

* Improve code clarity; minor refactoring.

* lint

---------

Co-authored-by: Armin Samii <[email protected]>
Co-authored-by: HEdingfield <[email protected]>
  • Loading branch information
@artoonie @HEdingfield
3 people authored Jun 23, 2024
1 parent 9db43dc commit 04a3835
Show file tree
Hide file tree
Showing 3 changed files with 89 additions and 83 deletions.
2 changes: 1 addition & 1 deletion src/main/java/network/brightspots/rcv/ResultsWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -554,7 +554,7 @@ void generateOverallSummaryFiles(
// Note that the castVoteRecords list MUST be stable, as cvrSourceData
// relies on its exact ordering to determine which source each record came from.
// Returns the filepath written
String writeRctabCvrCsv(
String writeRcTabCvrCsv(
List<CastVoteRecord> castVoteRecords,
List<CvrSourceData> cvrSourceData,
String csvOutputFolder)
Expand Down
106 changes: 56 additions & 50 deletions src/main/java/network/brightspots/rcv/TabulatorSession.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
import java.util.Map;
import java.util.Set;
import java.util.function.BiConsumer;
import javafx.util.Pair;
import network.brightspots.rcv.CastVoteRecord.CvrParseException;
import network.brightspots.rcv.ContestConfig.Provider;
import network.brightspots.rcv.ContestConfig.UnrecognizedProviderException;
Expand Down Expand Up @@ -105,11 +104,12 @@ boolean convertToCdf(BiConsumer<Double, Double> progressUpdate) {

Progress progress = new Progress(config, 0, progressUpdate);

if (setUpLogging(config.getOutputDirectory()) && config.validate().isEmpty()) {
if (setUpLogging(config.getOutputDirectory())
&& config.validate().isEmpty()) {
Logger.info("Converting CVR(s) to CDF...");
try {
FileUtils.createOutputDirectory(config.getOutputDirectory());
LoadedCvrData castVoteRecords = parseCastVoteRecords(config, progress);
LoadedCvrData castVoteRecords = parseCastVoteRecords(config, progress, false);
if (!castVoteRecords.successfullyReadAll) {
Logger.severe("Aborting conversion due to cast vote record errors!");
} else {
Expand Down Expand Up @@ -145,24 +145,26 @@ boolean convertToCdf(BiConsumer<Double, Double> progressUpdate) {
return conversionSuccess;
}

boolean convertToCdf() {
return convertToCdf(null);
void convertToCdf() {
convertToCdf(null);
}

LoadedCvrData parseAndCountCastVoteRecords(BiConsumer<Double, Double> progressUpdate)
throws CastVoteRecordGenericParseException {
ContestConfig config = ContestConfig.loadContestConfig(configPath);
Progress progress = new Progress(config, 0, progressUpdate);
return parseCastVoteRecords(config, progress);
return parseCastVoteRecords(config, progress, false);
}

// Returns a List of exception class names that were thrown while tabulating.
// Operator name is required for the audit logs.
// Note: An exception MUST be returned any time tabulation does not run.
// In general, that means any Logger.severe in this function should be accompanied
// by an exceptionsEncountered.add(...) call.
List<String> tabulate(String operatorName, LoadedCvrData expectedCvrData,
BiConsumer<Double, Double> progressUpdate) {
List<String> tabulate(
String operatorName,
LoadedCvrData expectedCvrData,
BiConsumer<Double, Double> progressUpdate) {
Logger.info("Starting tabulation session...");
List<String> exceptionsEncountered = new LinkedList<>();
ContestConfig config = ContestConfig.loadContestConfig(configPath);
Expand Down Expand Up @@ -208,9 +210,9 @@ List<String> tabulate(String operatorName, LoadedCvrData expectedCvrData,
// Read cast vote records and slice IDs from CVR files
Set<String> newWinnerSet;
try {
LoadedCvrData castVoteRecords = parseCastVoteRecords(config, progress);
LoadedCvrData castVoteRecords = parseCastVoteRecords(config, progress, true);
if (config.getSequentialWinners().isEmpty()
&& !castVoteRecords.metadataMatches(expectedCvrData)) {
&& !castVoteRecords.metadataMatches(expectedCvrData)) {
Logger.severe("CVR data has changed between loading the CVRs and reading them!");
exceptionsEncountered.add(TabulationAbortedException.class.toString());
break;
Expand Down Expand Up @@ -245,7 +247,7 @@ List<String> tabulate(String operatorName, LoadedCvrData expectedCvrData,
// normal operation (not multi-pass IRV, a.k.a. sequential multi-seat)
// Read cast vote records and precinct IDs from CVR files
try {
LoadedCvrData castVoteRecords = parseCastVoteRecords(config, progress);
LoadedCvrData castVoteRecords = parseCastVoteRecords(config, progress, true);
if (!castVoteRecords.metadataMatches(expectedCvrData)) {
Logger.severe("CVR data has changed between loading the CVRs and reading them!");
exceptionsEncountered.add(TabulationAbortedException.class.toString());
Expand Down Expand Up @@ -278,7 +280,8 @@ List<String> tabulate(String operatorName) {
Set<String> loadSliceNamesFromCvrs(ContestConfig.TabulateBySlice slice, ContestConfig config) {
Progress progress = new Progress(config, 0, null);
try {
List<CastVoteRecord> castVoteRecords = parseCastVoteRecords(config, progress).getCvrs();
List<CastVoteRecord> castVoteRecords =
parseCastVoteRecords(config, progress, false).getCvrs();
return new Tabulator(castVoteRecords, config).getEnabledSliceIds().get(slice);
} catch (TabulationAbortedException | CastVoteRecordGenericParseException e) {
throw new RuntimeException(e);
Expand Down Expand Up @@ -318,11 +321,18 @@ private Set<String> runTabulationForConfig(
return winners;
}

// parse CVR files referenced in the ContestConfig object into a list of CastVoteRecords
// param: config object containing CVR file paths to parse
// returns: list of parsed CVRs or null if an error was encountered
private LoadedCvrData parseCastVoteRecords(ContestConfig config, Progress progress)
throws CastVoteRecordGenericParseException {
/**
* Parse CVR files referenced in the ContestConfig object into a list of CastVoteRecords.
*
* @param config Object containing CVR file paths to parse.
* @param progress Object tracking progress of parsing the CVRs.
* @param shouldOutputRcTabCvr Whether to output the simplified RCTab CVR CSV file.
* @return List of parsed CVRs or null if an error was encountered.
* @throws CastVoteRecordGenericParseException If any failure occurs when parsing CVRs.
*/
private LoadedCvrData parseCastVoteRecords(
ContestConfig config, Progress progress, boolean shouldOutputRcTabCvr)
throws CastVoteRecordGenericParseException {
Logger.info("Parsing cast vote records...");
List<CastVoteRecord> castVoteRecords = new ArrayList<>();
boolean encounteredSourceProblem = false;
Expand All @@ -332,7 +342,7 @@ private LoadedCvrData parseCastVoteRecords(ContestConfig config, Progress progre

// At each iteration of the following loop, we add records from another source file.
for (int sourceIndex = 0; sourceIndex < config.rawConfig.cvrFileSources.size(); ++sourceIndex) {
RawContestConfig.CvrSource source = config.rawConfig.cvrFileSources.get(sourceIndex);
RawContestConfig.CvrSource source = config.rawConfig.cvrFileSources.get(sourceIndex);
String cvrPath = config.resolveConfigPath(source.getFilePath());
Provider provider = ContestConfig.getProvider(source);
try {
Expand All @@ -342,12 +352,9 @@ private LoadedCvrData parseCastVoteRecords(ContestConfig config, Progress progre
reader.readCastVoteRecords(castVoteRecords);

// Update the per-source data for the results writer
cvrSourceData.add(new ResultsWriter.CvrSourceData(
source,
reader,
sourceIndex,
startIndex,
castVoteRecords.size() - 1));
cvrSourceData.add(
new ResultsWriter.CvrSourceData(
source, reader, sourceIndex, startIndex, castVoteRecords.size() - 1));

// Check for unrecognized candidates
Map<String, Integer> unrecognizedCandidateCounts =
Expand Down Expand Up @@ -407,16 +414,18 @@ private LoadedCvrData parseCastVoteRecords(ContestConfig config, Progress progre
Logger.info("Parsed %d cast vote records successfully.", castVoteRecords.size());

// Output the RCTab-CSV CVR
try {
ResultsWriter writer =
if (shouldOutputRcTabCvr) {
try {
ResultsWriter writer =
new ResultsWriter().setContestConfig(config).setTimestampString(timestampString);
this.convertedFilePath =
writer.writeRctabCvrCsv(
this.convertedFilePath =
writer.writeRcTabCvrCsv(
castVoteRecords,
cvrSourceData,
config.getOutputDirectory());
} catch (IOException exception) {
// error already logged in ResultsWriter
} catch (IOException exception) {
// error already logged in ResultsWriter
}
}
}
}
Expand All @@ -438,27 +447,25 @@ static class UnrecognizedCandidatesException extends Exception {
}
}

static class CastVoteRecordGenericParseException extends Exception {
}
static class CastVoteRecordGenericParseException extends Exception {}

/**
* A summary of the cast vote records that have been read.
* Manages CVR in memory, so you can retain metadata about the loaded CVRs without
* keeping them all in memory. Use .discard() to free up memory. After memory is freed,
* all other operations except for getCvrs() are still valid.
* A summary of the cast vote records that have been read. Manages CVR in memory, so you can
* retain metadata about the loaded CVRs without keeping them all in memory. Use .discard() to
* free up memory. After memory is freed, all other operations except for getCvrs() are still
* valid.
*/
public static class LoadedCvrData {
public static final LoadedCvrData MATCHES_ALL = new LoadedCvrData();
public final boolean successfullyReadAll;

private List<CastVoteRecord> cvrs;
private final int numCvrs;
private List<ResultsWriter.CvrSourceData> cvrSourcesData;
private final List<ResultsWriter.CvrSourceData> cvrSourcesData;
private boolean isDiscarded;
private final boolean doesMatchAllMetadata;

public LoadedCvrData(List<CastVoteRecord> cvrs,
List<ResultsWriter.CvrSourceData> cvrSourcesData) {
LoadedCvrData(List<CastVoteRecord> cvrs, List<ResultsWriter.CvrSourceData> cvrSourcesData) {
this.cvrs = cvrs;
this.successfullyReadAll = cvrs != null;
this.numCvrs = cvrs != null ? cvrs.size() : 0;
Expand All @@ -468,8 +475,8 @@ public LoadedCvrData(List<CastVoteRecord> cvrs,
}

/**
* This constructor will cause metadataMatches to always return true,
* and contains no true statistics.
* This constructor will cause metadataMatches to always return true, and contains no true
* statistics.
*/
private LoadedCvrData() {
this.cvrs = null;
Expand All @@ -481,23 +488,23 @@ private LoadedCvrData() {
}

/**
* Currently only checks if the number of CVRs matches, but can be extended to ensure
* exact matches or meet other needs.
* Currently only checks if the number of CVRs matches, but can be extended to ensure exact
* matches or meet other needs.
*
* @param other The loaded CVRs to compare against
* @return whether the metadata matches
*/
public boolean metadataMatches(LoadedCvrData other) {
return other.doesMatchAllMetadata
|| this.doesMatchAllMetadata
|| other.numCvrs() == this.numCvrs();
|| this.doesMatchAllMetadata
|| other.numCvrs() == this.numCvrs();
}

public int numCvrs() {
return numCvrs;
}

public List<ResultsWriter.CvrSourceData> getCvrSourcesData() {
List<ResultsWriter.CvrSourceData> getCvrSourcesData() {
return cvrSourcesData;
}

Expand All @@ -506,7 +513,7 @@ public void discard() {
isDiscarded = true;
}

public List<CastVoteRecord> getCvrs() {
List<CastVoteRecord> getCvrs() {
if (isDiscarded) {
throw new IllegalStateException("CVRs have been discarded from memory.");
}
Expand All @@ -516,11 +523,10 @@ public List<CastVoteRecord> getCvrs() {
public void printSummary() {
Logger.info("Cast Vote Record summary:");
for (ResultsWriter.CvrSourceData sourceData : cvrSourcesData) {
Logger.info("Source %d: %s",
sourceData.sourceIndex + 1, sourceData.source.getFilePath());
Logger.info("Source %d: %s", sourceData.sourceIndex + 1, sourceData.source.getFilePath());
Logger.info(" uses provider: %s", sourceData.source.getProvider());
Logger.info(" read %d cast vote records", sourceData.getNumCvrs());
}
}
}
}
}
64 changes: 32 additions & 32 deletions ...tspots/rcv/test_data/csv_missing_header_test/csv_missing_header_test_expected_summary.csv
View file
Original file line number Diff line number Diff line change
@@ -1,32 +1,32 @@
Contest Information
Generated By,RCTab 1.3.999
CSV Format Version,1
Type of Election,Single-Winner
Contest,Missing Header CSV Test
Jurisdiction,"Portland, ME"
Office,Mayor
Date,2024-06-14
Winner(s),Cucumber
Final Threshold,14
Contest Summary
Number to be Elected,1
Number of Candidates,5
Total Number of Ballots,26
Number of Undervotes,0
Rounds,Round 1 Votes,% of vote,transfer,Round 2 Votes,% of vote,transfer,Round 3 Votes,% of vote,transfer,Round 4 Votes,% of vote,transfer
Eliminated,Undeclared Write-ins,,,Broccoli,,,Cauliflower,,,,,
Elected,,,,,,,,,,Cucumber,,
Cucumber,8,30.76%,2,10,38.46%,0,10,38.46%,4,14,53.84%,0
Lettuce,5,19.23%,2,7,26.92%,3,10,38.46%,2,12,46.15%,0
Cauliflower,4,15.38%,1,5,19.23%,1,6,23.07%,-6,0,0.0%,0
Broccoli,3,11.53%,1,4,15.38%,-4,0,0.0%,0,0,0.0%,0
Undeclared Write-ins,6,23.07%,-6,0,0.0%,0,0,0.0%,0,0,0.0%,0
Active Ballots,26,,,26,,,26,,,26,,
Current Round Threshold,14,,,14,,,14,,,14,,
Inactive Ballots by Overvotes,0,,0,0,,0,0,,0,0,,0
Inactive Ballots by Skipped Rankings,0,,0,0,,0,0,,0,0,,0
Inactive Ballots by Exhausted Choices,0,,0,0,,0,0,,0,0,,0
Inactive Ballots by Repeated Rankings,0,,0,0,,0,0,,0,0,,0
Inactive Ballots Total,0,,0,0,,0,0,,0,0,,0
Contest Information
Generated By,RCTab 1.3.999
CSV Format Version,1
Type of Election,Single-Winner
Contest,Missing Header CSV Test
Jurisdiction,"Portland, ME"
Office,Mayor
Date,2024-06-14
Winner(s),Cucumber
Final Threshold,14

Contest Summary
Number to be Elected,1
Number of Candidates,5
Total Number of Ballots,26
Number of Undervotes,0

Rounds,Round 1 Votes,% of vote,transfer,Round 2 Votes,% of vote,transfer,Round 3 Votes,% of vote,transfer,Round 4 Votes,% of vote,transfer
Eliminated,Undeclared Write-ins,,,Broccoli,,,Cauliflower,,,,,
Elected,,,,,,,,,,Cucumber,,
Cucumber,8,30.76%,2,10,38.46%,0,10,38.46%,4,14,53.84%,0
Lettuce,5,19.23%,2,7,26.92%,3,10,38.46%,2,12,46.15%,0
Cauliflower,4,15.38%,1,5,19.23%,1,6,23.07%,-6,0,0.0%,0
Broccoli,3,11.53%,1,4,15.38%,-4,0,0.0%,0,0,0.0%,0
Undeclared Write-ins,6,23.07%,-6,0,0.0%,0,0,0.0%,0,0,0.0%,0
Active Ballots,26,,,26,,,26,,,26,,
Current Round Threshold,14,,,14,,,14,,,14,,
Inactive Ballots by Overvotes,0,,0,0,,0,0,,0,0,,0
Inactive Ballots by Skipped Rankings,0,,0,0,,0,0,,0,0,,0
Inactive Ballots by Exhausted Choices,0,,0,0,,0,0,,0,0,,0
Inactive Ballots by Repeated Rankings,0,,0,0,,0,0,,0,0,,0
Inactive Ballots Total,0,,0,0,,0,0,,0,0,,0

0 comments on commit 04a3835

Please sign in to comment.