Check for case where CSV Row has too many columns

snowfakery/standard_plugins/datasets.py

@@ -9,7 +9,7 @@
 from sqlalchemy.sql.elements import quoted_name
 
 from yaml.representer import Representer
-from snowfakery.data_gen_exceptions import DataGenNameError
+from snowfakery.data_gen_exceptions import DataGenError, DataGenNameError
 
 from snowfakery.plugins import (
     SnowfakeryPlugin,
@@ -113,12 +113,22 @@ def __init__(self, datasource: Path, repeat: bool):
     def start(self):
         self.file.seek(0)
         d = DictReader(self.file)
-        self.results = (DatasetPluginResult(row) for row in d)
+
+        plugin_result = self.plugin_result
+        self.results = (plugin_result(row) for row in d)
 
     def close(self):
         self.results = None
         self.file.close()
 
+    def plugin_result(self, row):
+        if None in row:
+            raise DataGenError(
+                f"Your CSV row has more columns than the CSV header:  {row[None]}, {self.datasource}"
+            )
+
+        return DatasetPluginResult(row)
+
 
 class DatasetPluginResult(PluginResult):
     def __getattr__(self, name):

tests/badcsv_too_many_columns.csv

@@ -0,0 +1,5 @@
+id,FirstName,LastName
+0032D00000V6UvUQAV,Michael,Bluth,32252 Marc Mall Suite 349,South Kristenview,Massachusetts,65450,United States
+032D00000V6UvVQAV,Isabella,Wright,41497 Henson Motorway,West Marisaland,Alaska,10293,United States
+032D00000V6UvfQAF,Desiree,Shelton,84504 Darren Knolls Suite 023,Port Sandra,Pennsylvania,68863,United States
+032D00000V6UvkQAF,Deanna,Mcdaniel,838 Montoya Circle Apt. 857,West Robert,Louisiana,03074,United States

tests/test_external_datasets.py

@@ -187,6 +187,24 @@ def test_csv_bad_column_name(self):
             generate(StringIO(yaml), {})
         assert "'xname ', 'fake'" in str(e.value)
 
+    def test_csv_row_has_too_many_columns(self):
+        abs_path = str(Path(__file__).parent)
+        yaml = (
+            """
+        - plugin: snowfakery.standard_plugins.datasets.Dataset
+        - object: XXX
+          fields:
+            __address_from_csv:
+              Dataset.iterate:
+                dataset: %s/badcsv_too_many_columns.csv
+            foo: ${{__address_from_csv.name}}
+        """
+            % abs_path
+        )
+        with pytest.raises(exc.DataGenError) as e:
+            generate(StringIO(yaml), {})
+        assert "more columns" in str(e.value)
+
     def test_csv_utf_8_bom(self, generated_rows):
         abs_path = Path(__file__).parent
         yaml = (