io.metadata: Fix newline handling when reading CSV/TSV

CHANGES.md

@@ -2,6 +2,13 @@
 
 ## __NEXT__
 
+### Bug Fixes
+
+* Embedded newlines in quoted field values of metadata files are now properly handled. [#1561][] (@tsibley)
+
+[#1561]: https://github.com/nextstrain/augur/pull/1561
+
+
 
 ## 25.2.0 (24 July 2024)
 

augur/io/metadata.py

@@ -603,7 +603,7 @@ def __init__(self, path: str, delimiters: Sequence[str], id_columns: Sequence[st
 
     def open(self, **kwargs):
         """Open the file with auto-compression/decompression."""
-        return open_file(self.path, **kwargs)
+        return open_file(self.path, newline='', **kwargs)
 
     def _find_first(self, columns: Sequence[str]):
         """Return the first column in `columns` that is present in the metadata.
@@ -646,7 +646,7 @@ def _get_delimiter(path: str, valid_delimiters: Iterable[str]):
         if len(delimiter) != 1:
             raise AugurError(f"Delimiters must be single-character strings. {delimiter!r} does not satisfy that condition.")
 
-    with open_file(path) as file:
+    with open_file(path, newline='') as file:
         try:
             # Infer the delimiter from the first line.
             return csv.Sniffer().sniff(file.readline(), "".join(valid_delimiters)).delimiter

tests/io/test_metadata.py

@@ -510,7 +510,7 @@ def test_write_records_to_tsv_with_empty_records(self, tmpdir):
 
 def write_lines(tmpdir, lines):
     path = str(tmpdir / "tmp")
-    with open(path, 'w') as f:
+    with open(path, 'w', newline='') as f:
         f.writelines(lines)
     return path
 
@@ -603,3 +603,20 @@ def test_rows_strict_missing(self, tmpdir):
             {'a': '2', 'b': '2', 'c': ''},
             {'a': '3', 'b': '2', 'c': None},
         ]
+
+    def test_rows_embedded_newline(self, tmpdir):
+        """Test behavior when reading rows with an embedded newline."""
+        path = write_lines(tmpdir, [
+            'a,b,c\n',
+            '1,2,3\n',
+            '4,"5\r\n6",7\n',
+            '8,9,10\n',
+        ])
+
+        m = Metadata(path, delimiters=',', id_columns=['a'])
+
+        assert list(m.rows()) == [
+            {'a': '1', 'b': '2', 'c': '3'},
+            {'a': '4', 'b': '5\r\n6', 'c': '7'},
+            {'a': '8', 'b': '9', 'c': '10'},
+        ]