Make DataFrame materializer types optional

Allow any combination of materializer types to be specified, with no specific type required.
dagster-io · Aug 11, 2020 · ea1758b · ea1758b
1 parent ec9dbd6
commit ea1758b
Showing 1 changed file with 18 additions and 8 deletions.
diff --git a/python_modules/libraries/dagster-dask/dagster_dask/data_frame.py b/python_modules/libraries/dagster-dask/dagster_dask/data_frame.py
@@ -37,7 +37,7 @@ def dict_without_keys(ddict, *keys):
 @dagster_type_materializer(
     Shape({
         'to': {
-            'csv': Permissive(
+            'csv': Field(
                 {
                     'path': Field(
                         Any,
@@ -103,9 +103,10 @@ def dict_without_keys(ddict, *keys):
                         is_required=False,
                         description="Options to be passed in to the compute method",
                     ),
-                }
+                },
+                is_required=False,
             ),
-            'parquet': Permissive(
+            'parquet': Field(
                 {
                     'path': Field(
                         Any,
@@ -185,9 +186,10 @@ def dict_without_keys(ddict, *keys):
                         is_required=False,
                         description="Options to be passed in to the compute method.",
                     ),
-                }
+                },
+                is_required=False,
             ),
-            'hdf': Permissive(
+            'hdf': Field(
                 {
                     'path': Field(
                         Any,
@@ -219,9 +221,10 @@ def dict_without_keys(ddict, *keys):
                         is_required=False,
                         description="The scheduler to use, like 'threads' or 'processes'.",
                     ),
-                }
+                },
+                is_required=False,
             ),
-            'json': Permissive(
+            'json': Field(
                 {
                     'path': Field(
                         Any,
@@ -244,6 +247,11 @@ def dict_without_keys(ddict, *keys):
                         is_required=False,
                         description="default is 'strict', how to respond to errors in the conversion (see ``str.encode()``).",
                     ),
+                    'orient': Field(
+                        String,
+                        is_required=False,
+                        description="if not specified, lines=True when orient=’records’, False otherwise."
+                    ),
                     'storage_options': Field(
                         Permissive(),
                         is_required=False,
@@ -266,8 +274,9 @@ def dict_without_keys(ddict, *keys):
                         String, is_required=False, description="String like 'gzip' or 'xz'.",
                     ),
                 },
+                is_required=False,
             ),
-            'sql': Permissive(
+            'sql': Field(
                 {
                     'name': Field(String, is_required=True, description="Name of SQL table",),
                     'uri': Field(
@@ -357,6 +366,7 @@ def dict_without_keys(ddict, *keys):
                         """,
                     ),
                 },
+                is_required=False,
             ),
         },
         'sample': Field(