decoding errors test

enso-org · May 6, 2022 · c5ac70f · c5ac70f
1 parent 3d2d205
commit c5ac70f
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 2 deletions.
diff --git a/std-bits/table/src/main/java/org/enso/table/read/DelimitedReader.java b/std-bits/table/src/main/java/org/enso/table/read/DelimitedReader.java
@@ -3,7 +3,6 @@
 import com.univocity.parsers.csv.CsvFormat;
 import com.univocity.parsers.csv.CsvParser;
 import com.univocity.parsers.csv.CsvParserSettings;
-import java.io.InputStream;
 import java.io.Reader;
 import java.util.ArrayList;
 import java.util.Arrays;

diff --git a/test/Table_Tests/data/utf16_invalid.csv b/test/Table_Tests/data/utf16_invalid.csv
diff --git a/test/Table_Tests/src/Delimited_Read_Spec.enso b/test/Table_Tests/src/Delimited_Read_Spec.enso
@@ -91,7 +91,26 @@ spec =
             table.at 'ć😎' . to_vector . should_equal ['แมวมีสี่ขา']
 
         Test.specify "should report errors when encountering malformed characters" <|
-            Nothing
+            action_1 on_problems =
+                (File_Format.Delimited "," headers=True).read (Enso_Project.data / "utf8_invalid.csv") on_problems
+            tester_1 table =
+                table.columns.map .name . should_equal ['a', 'b', 'c']
+                table.at 'a' . to_vector . should_equal ['ą']
+                table.at 'b' . to_vector . should_equal ['\uFFFF']
+                table.at 'c' . to_vector . should_equal ['\uFFFD(']
+            problems_1 = [Encoding_Error "Encoding issues at TODO"]
+            Problems.test_problem_handling action_1 problems_1 tester_1
+
+            action_2 on_problems =
+                (File_Format.Delimited "," headers=True encoding=Encoding.utf_16_be).read (Enso_Project.data / "utf16_invalid.csv") on_problems
+            tester_2 table =
+                table.columns.map .name . should_equal ['a', 'b', 'c']
+                table.at 'a' . to_vector . should_equal ['ą']
+                table.at 'b' . to_vector . should_equal ['\uFFFF']
+                table.at 'c' . to_vector . should_equal ['\uFFFD']
+            problems_2 = [Encoding_Error "Encoding issues at TODO"]
+            Problems.test_problem_handling action_2 problems_2 tester_2
+
 
         Test.specify "should handle duplicated columns" <|
             table = (File_Format.Delimited "," headers=True).read (Enso_Project.data / "duplicated_columns.csv") Problem_Behavior.Report_Error