Csv export refinement

src/main/java/au/org/democracydevelopers/raireservice/service/GetAssertionsCsvService.java

@@ -31,6 +31,7 @@ the raire assertion generation engine (https://github.com/DemocracyDevelopers/ra
 import static au.org.democracydevelopers.raireservice.service.Metadata.extremumHeaders;
 import static au.org.democracydevelopers.raireservice.service.Metadata.csvHeaders;
 import static au.org.democracydevelopers.raireservice.util.CSVUtils.escapeThenJoin;
+import static au.org.democracydevelopers.raireservice.util.CSVUtils.intListToString;
 
 import au.org.democracydevelopers.raireservice.persistence.entity.Assertion;
 import au.org.democracydevelopers.raireservice.persistence.repository.AssertionRepository;
@@ -186,8 +187,7 @@ private record extremumResult<T>(
      * @return a CSV row with the relevant data, as a string.
      */
     String toCSVRow() {
-      return escapeThenJoin(List.of(statisticName, value.toString(),
-          String.join(", ", indices.stream().map(Object::toString).toList())));
+      return escapeThenJoin(List.of(statisticName, value.toString()))+","+intListToString(indices);
     }
   }
 
@@ -273,7 +273,7 @@ private String makeContents(List<Assertion> assertions) {
     List<String> rows = new ArrayList<>();
 
     for (Assertion assertion : assertions) {
-      rows.add(index++ + ", " + escapeThenJoin(assertion.asCSVRow()));
+      rows.add(index++ + "," + escapeThenJoin(assertion.asCSVRow()));
     }
 
     return String.join("\n", rows) + "\n";

src/main/java/au/org/democracydevelopers/raireservice/util/CSVUtils.java

@@ -38,6 +38,24 @@ public class CSVUtils {
    *         in one cell.
    */
   public static String escapeThenJoin(List<String> data) {
-    return String.join(", ", data.stream().map(StringEscapeUtils::escapeCsv).toList());
+    return String.join(",", data.stream().map(StringEscapeUtils::escapeCsv).toList());
+  }
+
+  /**
+   * Turn a list of integers into a pretty-printed string intended for a csv cell. This puts quotes
+   * around everything, including a singleton list, so that it is interpreted as a string by the
+   * spreadsheet/csv reader.
+   * @param data a list of Integers.
+   * @return the list as a pretty-printed comma-separated string with explicit quotes around it.
+   */
+  public static String intListToString(List<Integer> data) {
+    if(data.size() == 1) {
+      // Put quotes around it so it will be interpreted as a string by the csv parser
+      return "\""+data.getFirst()+"\"";
+    } else {
+      // This will return an empty string for an empty input list, otherwise a string with the data
+      // items comma-separated.
+      return "\""+String.join(", ", data.stream().map(Object::toString).toList())+"\"";
+    }
   }
 }

src/test/java/au/org/democracydevelopers/raireservice/controller/GetAssertionsCSVAPITests.java

@@ -52,6 +52,7 @@ the raire assertion generation engine (https://github.com/DemocracyDevelopers/ra
  * Contests which will be used for validity testing are preloaded into the database using
  * src/test/resources/simple_assertions_csv_challenges.sql.
  */
+
 @ActiveProfiles("csv-challenges")
 @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @AutoConfigureTestDatabase(replace = Replace.NONE)
@@ -101,33 +102,32 @@ public void testValidRequestWithNoAssertions() {
     assertTrue(response.getStatusCode().is2xxSuccessful());
     String output = response.getBody();
 
-    assertNotNull(output);
-    assertTrue(output.contains("Contest name, Lots of assertions with ties Contest\n"));
-    assertTrue(output.contains("Candidates, \"Alice, Bob, Chuan, Diego\"\n\n"));
-    assertTrue(output.contains("Extreme item, Value, Assertion IDs"));
-    assertTrue(output.contains("Margin, 220, \"2, 5, 6\""));
-    assertTrue(output.contains("Diluted margin, 0.22, \"2, 5, 6\""));
-    assertTrue(output.contains("Raire difficulty, 3.1, 3"));
-    assertTrue(output.contains("Current risk, 0.23, \"2, 3\""));
-    assertTrue(output.contains("Optimistic samples to audit, 910, 4"));
-    assertTrue(output.contains("Estimated samples to audit, 430, \"2, 5\"\n\n"));
+    assertTrue(output.contains("Contest name,Lots of assertions with ties Contest\n"));
+    assertTrue(output.contains("Candidates,\"Alice,Bob,Chuan,Diego\"\n\n"));
+    assertTrue(output.contains("Extreme item,Value,Assertion IDs"));
+    assertTrue(output.contains("Margin,220,\"2, 5, 6\""));
+    assertTrue(output.contains("Diluted margin,0.22,\"2, 5, 6\""));
+    assertTrue(output.contains("Raire difficulty,3.1,\"3\""));
+    assertTrue(output.contains("Current risk,0.23,\"2, 3\""));
+    assertTrue(output.contains("Optimistic samples to audit,910,\"4\""));
+    assertTrue(output.contains("Estimated samples to audit,430,\"2, 5\"\n\n"));
     assertTrue(output.contains(
-        "ID, Type, Winner, Loser, Assumed continuing, Difficulty, Margin, Diluted margin, Risk, "
-        + "Estimated samples to audit, Optimistic samples to audit, Two vote over count, "
-        + "One vote over count, Other discrepancy count, One vote under count, "
+        "ID,Type,Winner,Loser,Assumed continuing,Difficulty,Margin,Diluted margin,Risk,"
+        + "Estimated samples to audit,Optimistic samples to audit,Two vote over count,"
+        + "One vote over count,Other discrepancy count,One vote under count,"
         + "Two vote under count\n"
     ));
-    assertTrue(output.contains("1, NEB, Alice, Bob, , 2.1, 320, 0.32, 0.04, 110, 100, 0, 0, 0, 0, 0\n"));
-    assertTrue(output.contains("2, NEB, Chuan, Bob, , 1.1, 220, 0.22, 0.23, 430, 200, 0, 0, 0, 0, 0\n"));
-    assertTrue(output.contains("3, NEB, Diego, Chuan, , 3.1, 320, 0.32, 0.23, 50, 110, 0, 0, 0, 0, 0\n"));
+    assertTrue(output.contains("1,NEB,Alice,Bob,,2.1,320,0.32,0.04,110,100,0,0,0,0,0\n"));
+    assertTrue(output.contains("2,NEB,Chuan,Bob,,1.1,220,0.22,0.23,430,200,0,0,0,0,0\n"));
+    assertTrue(output.contains("3,NEB,Diego,Chuan,,3.1,320,0.32,0.23,50,110,0,0,0,0,0\n"));
     assertTrue(output.contains(
-        "4, NEN, Alice, Bob, \"Alice, Bob, Chuan\", 2.0, 420, 0.42, 0.04, 320, 910, 0, 0, 0, 0, 0\n"
+        "4,NEN,Alice,Bob,\"Alice,Bob,Chuan\",2.0,420,0.42,0.04,320,910,0,0,0,0,0\n"
     ));
     assertTrue(output.contains(
-        "5, NEN, Alice, Diego, \"Alice, Diego\", 1.1, 220, 0.22, 0.07, 430, 210, 0, 0, 0, 0, 0\n"
+        "5,NEN,Alice,Diego,\"Alice,Diego\",1.1,220,0.22,0.07,430,210,0,0,0,0,0\n"
     ));
     assertTrue(output.contains(
-        "6, NEN, Alice, Bob, \"Alice, Bob, Diego\", 1.2, 220, 0.22, 0.04, 400, 110, 0, 0, 0, 0, 0\n"
+        "6,NEN,Alice,Bob,\"Alice,Bob,Diego\",1.2,220,0.22,0.04,400,110,0,0,0,0,0\n"
     ));
   }
 
@@ -170,24 +170,23 @@ public void testCSVDemoContest() {
     ResponseEntity<String> response = restTemplate.postForEntity(url, request, String.class);
     String output = response.getBody();
 
-    assertNotNull(output);
-    assertTrue(output.contains("Contest name, CSV Demo Contest\n"));
-    assertTrue(output.contains("Candidates, \"Alice, Bob, Chuan, Diego\"\n\n"));
-    assertTrue(output.contains("Extreme item, Value, Assertion IDs\n"));
-    assertTrue(output.contains("Margin, 100, 2\n"));
-    assertTrue(output.contains("Diluted margin, 0.1, 2\n"));
-    assertTrue(output.contains("Raire difficulty, 6.1, 2\n"));
-    assertTrue(output.contains("Current risk, 0.06, 1\n"));
-    assertTrue(output.contains("Optimistic samples to audit, 45, 2\n"));
-    assertTrue(output.contains("Estimated samples to audit, 55, 1\n"));
+    assertTrue(output.contains("Contest name,CSV Demo Contest\n"));
+    assertTrue(output.contains("Candidates,\"Alice,Bob,Chuan,Diego\"\n\n"));
+    assertTrue(output.contains("Extreme item,Value,Assertion IDs\n"));
+    assertTrue(output.contains("Margin,100,\"2\"\n"));
+    assertTrue(output.contains("Diluted margin,0.1,\"2\"\n"));
+    assertTrue(output.contains("Raire difficulty,6.1,\"2\"\n"));
+    assertTrue(output.contains("Current risk,0.06,\"1\"\n"));
+    assertTrue(output.contains("Optimistic samples to audit,45,\"2\"\n"));
+    assertTrue(output.contains("Estimated samples to audit,55,\"1\"\n"));
     assertTrue(output.contains(
-        "ID, Type, Winner, Loser, Assumed continuing, Difficulty, Margin, Diluted margin, Risk, "
-            + "Estimated samples to audit, Optimistic samples to audit, Two vote over count, "
-            + "One vote over count, Other discrepancy count, One vote under count, "
+        "ID,Type,Winner,Loser,Assumed continuing,Difficulty,Margin,Diluted margin,Risk,"
+            + "Estimated samples to audit,Optimistic samples to audit,Two vote over count,"
+            + "One vote over count,Other discrepancy count,One vote under count,"
             + "Two vote under count\n"));
-    assertTrue(output.contains("1, NEB, Bob, Alice, , 5.1, 112, 0.112, 0.06, 55, 35, 0, 2, 0, 0, 0\n"));
+    assertTrue(output.contains("1,NEB,Bob,Alice,,5.1,112,0.112,0.06,55,35,0,2,0,0,0\n"));
     assertTrue(output.contains(
-        "2, NEN, Diego, Chuan, \"Alice, Chuan, Diego\", 6.1, 100, 0.1, 0.05, 45, 45, 0, 0, 0, 0, 0\n"
+        "2,NEN,Diego,Chuan,\"Alice,Chuan,Diego\",6.1,100,0.1,0.05,45,45,0,0,0,0,0\n"
     ));
   }
 }

src/test/java/au/org/democracydevelopers/raireservice/service/GetAssertionsCSVTests.java

@@ -77,42 +77,42 @@ public void testCSVTies() throws RaireServiceException {
     GetAssertionsRequest request = new GetAssertionsRequest(
         "Lots of assertions with ties Contest", candidates, new BigDecimal("0.10"));
     String output = getAssertionsCSVService.generateCSV(request);
-    assertTrue(output.contains("Contest name, Lots of assertions with ties Contest\n"));
-    assertTrue(output.contains("Candidates, \"Alice, Bob, Chuan, Diego\"\n\n"));
-    assertTrue(output.contains("Extreme item, Value, Assertion IDs"));
-    assertTrue(output.contains("Margin, 220, \"2, 5, 6\""));
-    assertTrue(output.contains("Diluted margin, 0.22, \"2, 5, 6\""));
-    assertTrue(output.contains("Raire difficulty, 3.1, 3"));
-    assertTrue(output.contains("Current risk, 0.23, \"2, 3\""));
-    assertTrue(output.contains("Optimistic samples to audit, 910, 4"));
-    assertTrue(output.contains("Estimated samples to audit, 430, \"2, 5\"\n\n"));
+    assertTrue(output.contains("Contest name,Lots of assertions with ties Contest\n"));
+    assertTrue(output.contains("Candidates,\"Alice,Bob,Chuan,Diego\"\n\n"));
+    assertTrue(output.contains("Extreme item,Value,Assertion IDs"));
+    assertTrue(output.contains("Margin,220,\"2, 5, 6\""));
+    assertTrue(output.contains("Diluted margin,0.22,\"2, 5, 6\""));
+    assertTrue(output.contains("Raire difficulty,3.1,\"3\""));
+    assertTrue(output.contains("Current risk,0.23,\"2, 3\""));
+    assertTrue(output.contains("Optimistic samples to audit,910,\"4\""));
+    assertTrue(output.contains("Estimated samples to audit,430,\"2, 5\"\n\n"));
     assertTrue(output.contains(
-        "ID, Type, Winner, Loser, Assumed continuing, Difficulty, Margin, Diluted margin, Risk, "
-        + "Estimated samples to audit, Optimistic samples to audit, Two vote over count, "
-        + "One vote over count, Other discrepancy count, One vote under count, "
+        "ID,Type,Winner,Loser,Assumed continuing,Difficulty,Margin,Diluted margin,Risk,"
+        + "Estimated samples to audit,Optimistic samples to audit,Two vote over count,"
+        + "One vote over count,Other discrepancy count,One vote under count,"
         + "Two vote under count\n"
     ));
-    assertTrue(output.contains("1, NEB, Alice, Bob, , 2.1, 320, 0.32, 0.04, 110, 100, 0, 0, 0, 0, 0\n"));
-    assertTrue(output.contains("2, NEB, Chuan, Bob, , 1.1, 220, 0.22, 0.23, 430, 200, 0, 0, 0, 0, 0\n"));
-    assertTrue(output.contains("3, NEB, Diego, Chuan, , 3.1, 320, 0.32, 0.23, 50, 110, 0, 0, 0, 0, 0\n"));
+    assertTrue(output.contains("1,NEB,Alice,Bob,,2.1,320,0.32,0.04,110,100,0,0,0,0,0\n"));
+    assertTrue(output.contains("2,NEB,Chuan,Bob,,1.1,220,0.22,0.23,430,200,0,0,0,0,0\n"));
+    assertTrue(output.contains("3,NEB,Diego,Chuan,,3.1,320,0.32,0.23,50,110,0,0,0,0,0\n"));
     assertTrue(output.contains(
-        "4, NEN, Alice, Bob, \"Alice, Bob, Chuan\", 2.0, 420, 0.42, 0.04, 320, 910, 0, 0, 0, 0, 0\n"
+        "4,NEN,Alice,Bob,\"Alice,Bob,Chuan\",2.0,420,0.42,0.04,320,910,0,0,0,0,0\n"
     ));
     assertTrue(output.contains(
-        "5, NEN, Alice, Diego, \"Alice, Diego\", 1.1, 220, 0.22, 0.07, 430, 210, 0, 0, 0, 0, 0\n"
+        "5,NEN,Alice,Diego,\"Alice,Diego\",1.1,220,0.22,0.07,430,210,0,0,0,0,0\n"
     ));
     assertTrue(output.contains(
-        "6, NEN, Alice, Bob, \"Alice, Bob, Diego\", 1.2, 220, 0.22, 0.04, 400, 110, 0, 0, 0, 0, 0\n"
+        "6,NEN,Alice,Bob,\"Alice,Bob,Diego\",1.2,220,0.22,0.04,400,110,0,0,0,0,0\n"
     ));
   }
 
   /**
-   * Test for difficult characters in candidate names, including ' and " and ,
+   * Test for difficult characters in candidate names,including ' and " and ,
    * @throws RaireServiceException if assertion database retrieval fails.
    */
   @Test
   public void testCharacterEscaping() throws RaireServiceException {
-    testUtils.log(logger, "testCharacterEscaping");
+    testUtils.log(logger,"testCharacterEscaping");
     GetAssertionsRequest request = new GetAssertionsRequest("Lots of tricky characters Contest",
         trickyCharacters, new BigDecimal("0.10"));
     String output = getAssertionsCSVService.generateCSV(request);
@@ -129,27 +129,27 @@ public void testCharacterEscaping() throws RaireServiceException {
    */
   @Test
   public void testCsvDemoContest() throws RaireServiceException {
-    testUtils.log(logger, "testCsvDemoContest");
+    testUtils.log(logger,"testCsvDemoContest");
     GetAssertionsRequest request = new GetAssertionsRequest(
         "CSV Demo Contest", candidates, new BigDecimal("0.10"));
     String output = getAssertionsCSVService.generateCSV(request);
-    assertTrue(output.contains("Contest name, CSV Demo Contest\n"));
-    assertTrue(output.contains("Candidates, \"Alice, Bob, Chuan, Diego\"\n\n"));
-    assertTrue(output.contains("Extreme item, Value, Assertion IDs\n"));
-    assertTrue(output.contains("Margin, 100, 2\n"));
-    assertTrue(output.contains("Diluted margin, 0.1, 2\n"));
-    assertTrue(output.contains("Raire difficulty, 6.1, 2\n"));
-    assertTrue(output.contains("Current risk, 0.06, 1\n"));
-    assertTrue(output.contains("Optimistic samples to audit, 45, 2\n"));
-    assertTrue(output.contains("Estimated samples to audit, 55, 1\n"));
+    assertTrue(output.contains("Contest name,CSV Demo Contest\n"));
+    assertTrue(output.contains("Candidates,\"Alice,Bob,Chuan,Diego\"\n\n"));
+    assertTrue(output.contains("Extreme item,Value,Assertion IDs\n"));
+    assertTrue(output.contains("Margin,100,\"2\"\n"));
+    assertTrue(output.contains("Diluted margin,0.1,\"2\"\n"));
+    assertTrue(output.contains("Raire difficulty,6.1,\"2\"\n"));
+    assertTrue(output.contains("Current risk,0.06,\"1\"\n"));
+    assertTrue(output.contains("Optimistic samples to audit,45,\"2\"\n"));
+    assertTrue(output.contains("Estimated samples to audit,55,\"1\"\n"));
     assertTrue(output.contains(
-        "ID, Type, Winner, Loser, Assumed continuing, Difficulty, Margin, Diluted margin, Risk, "
-            + "Estimated samples to audit, Optimistic samples to audit, Two vote over count, "
-            + "One vote over count, Other discrepancy count, One vote under count, "
+        "ID,Type,Winner,Loser,Assumed continuing,Difficulty,Margin,Diluted margin,Risk,"
+            + "Estimated samples to audit,Optimistic samples to audit,Two vote over count,"
+            + "One vote over count,Other discrepancy count,One vote under count,"
             + "Two vote under count\n"));
-    assertTrue(output.contains("1, NEB, Bob, Alice, , 5.1, 112, 0.112, 0.06, 55, 35, 0, 2, 0, 0, 0\n"));
+    assertTrue(output.contains("1,NEB,Bob,Alice,,5.1,112,0.112,0.06,55,35,0,2,0,0,0\n"));
     assertTrue(output.contains(
-        "2, NEN, Diego, Chuan, \"Alice, Chuan, Diego\", 6.1, 100, 0.1, 0.05, 45, 45, 0, 0, 0, 0, 0\n"
+        "2,NEN,Diego,Chuan,\"Alice,Chuan,Diego\",6.1,100,0.1,0.05,45,45,0,0,0,0,0\n"
     ));
   }
 }