Refactor CLI hint code and types, more CLI hint arguments

metrics/coverage_high_water_mark

@@ -1 +1 @@
-94.0200
+94.0100

metrics/mypy_high_water_mark

@@ -1 +1 @@
-92.1100
+92.0400

records_mover/db/bigquery/load_job_config_options.py

@@ -4,6 +4,7 @@
 from ...records.load_plan import RecordsLoadPlan
 from ...records.records_format import DelimitedRecordsFormat, ParquetRecordsFormat
 from records_mover.records import RecordsHints
+from records_mover.records.delimited import validate_partial_hints
 from google.cloud.bigquery.job import CreateDisposition, WriteDisposition
 from google.cloud import bigquery
 import logging
@@ -106,8 +107,10 @@ def load_job_config(unhandled_hints: Set[str],
     config.schema_update_options = None
 
     if isinstance(load_plan.records_format, DelimitedRecordsFormat):
+        hints = validate_partial_hints(load_plan.records_format.hints,
+                                       fail_if_cant_handle_hint=load_plan.processing_instructions.fail_if_cant_handle_hint)
         add_load_job_csv_config(unhandled_hints,
-                                load_plan.records_format.hints,
+                                hints,
                                 fail_if_cant_handle_hint,
                                 config)
         return config

records_mover/db/postgres/copy_options/__init__.py

@@ -1,5 +1,5 @@
 from records_mover.records.load_plan import RecordsLoadPlan
-from records_mover.records.delimited import RecordsHints
+from records_mover.records.delimited import ValidatedRecordsHints
 from records_mover.records.records_format import DelimitedRecordsFormat
 import logging
 from typing import Set, Tuple, Optional
@@ -16,7 +16,7 @@
 # https://www.postgresql.org/docs/9.2/sql-copy.html
 
 
-def needs_csv_format(hints: RecordsHints) -> bool:
+def needs_csv_format(hints: ValidatedRecordsHints) -> bool:
     # This format option is used for importing and exporting the Comma
     # Separated Value (CSV) file format used by many other programs,
     # such as spreadsheets. Instead of the escaping rules used by
@@ -31,7 +31,7 @@ def needs_csv_format(hints: RecordsHints) -> bool:
     # QUOTE character or the ESCAPE character is preceded by the
     # escape character. You can also use FORCE_QUOTE to force quotes
     # when outputting non-NULL values in specific columns.
-    if hints['header-row'] or (hints['quoting'] is not None):
+    if hints.header_row or (hints.quoting is not None):
         return True
 
     return False
@@ -44,7 +44,7 @@ def postgres_copy_to_options(unhandled_hints: Set[str],
                                          Tuple[DateOutputStyle,
                                                Optional[DateOrderStyle],
                                                PostgresCopyOptions]:
-    hints = delimited_records_format.hints
+    hints = delimited_records_format.validate(fail_if_cant_handle_hint=fail_if_cant_handle_hint)
 
     if needs_csv_format(hints):
         copy_options = postgres_copy_options_csv(unhandled_hints,
@@ -74,7 +74,7 @@ def postgres_copy_from_options(unhandled_hints: Set[str],
     if not isinstance(load_plan.records_format, DelimitedRecordsFormat):
         raise NotImplementedError("Not currently able to import "
                                   f"{load_plan.records_format.format_type}")
-    hints = load_plan.records_format.hints
+    hints = load_plan.records_format.validate(fail_if_cant_handle_hint=fail_if_cant_handle_hint)
 
     if needs_csv_format(hints):
         postgres_copy_options = postgres_copy_options_csv(unhandled_hints,

records_mover/db/postgres/copy_options/common.py

@@ -1,5 +1,6 @@
 from records_mover.utils import quiet_remove
-from records_mover.records.delimited import cant_handle_hint, RecordsHints
+from records_mover.records.records_format import DelimitedRecordsFormat
+from records_mover.records.delimited import cant_handle_hint, ValidatedRecordsHints
 from typing import Set
 from .types import PostgresCopyOptions
 
@@ -15,7 +16,7 @@
 
 
 def postgres_copy_options_common(unhandled_hints: Set[str],
-                                 hints: RecordsHints,
+                                 hints: ValidatedRecordsHints,
                                  fail_if_cant_handle_hint: bool,
                                  original_postgres_options: PostgresCopyOptions) ->\
         PostgresCopyOptions:
@@ -27,7 +28,7 @@ def postgres_copy_options_common(unhandled_hints: Set[str],
     #  this option is omitted, the current client encoding is
     #  used. See the Notes below for more details.
 
-    encoding_hint: str = hints['encoding']  # type: ignore
+    encoding_hint = hints.encoding
     if encoding_hint in postgres_encoding_names:
         postgres_options['encoding'] = postgres_encoding_names[encoding_hint]
         quiet_remove(unhandled_hints, 'encoding')
@@ -51,7 +52,7 @@ def postgres_copy_options_common(unhandled_hints: Set[str],
     #  is ignored. This option is allowed only when using CSV format.
     #
     quiet_remove(unhandled_hints, 'header-row')
-    postgres_options['header'] = hints['header-row']
+    postgres_options['header'] = hints.header_row
 
     # OIDS
     #
@@ -67,7 +68,7 @@ def postgres_copy_options_common(unhandled_hints: Set[str],
     #  format, a comma in CSV format. This must be a single one-byte
     #  character. This option is not allowed when using binary format.
     #
-    postgres_options['delimiter'] = hints['field-delimiter']
+    postgres_options['delimiter'] = hints.field_delimiter
     quiet_remove(unhandled_hints, 'field-delimiter')
 
     return postgres_options

records_mover/db/postgres/copy_options/csv.py

@@ -1,13 +1,14 @@
 from records_mover.utils import quiet_remove
-from records_mover.records.delimited import cant_handle_hint, RecordsHints
+from records_mover.records.records_format import DelimitedRecordsFormat
+from records_mover.records.delimited import cant_handle_hint, ValidatedRecordsHints
 from typing import Set
 from .mode import CopyOptionsMode
 from .common import postgres_copy_options_common
 from .types import PostgresCopyOptions, CopyOptionsModeType, _assert_never
 
 
 def postgres_copy_options_csv(unhandled_hints: Set[str],
-                              hints: RecordsHints,
+                              hints: ValidatedRecordsHints,
                               fail_if_cant_handle_hint: bool,
                               mode: CopyOptionsModeType) ->\
         PostgresCopyOptions:
@@ -41,7 +42,7 @@ def postgres_copy_options_csv(unhandled_hints: Set[str],
     #  format.
     #
 
-    postgres_options['quote'] = hints['quotechar']
+    postgres_options['quote'] = hints.quotechar
     quiet_remove(unhandled_hints, 'quotechar')
 
     # ESCAPE
@@ -53,12 +54,12 @@ def postgres_copy_options_csv(unhandled_hints: Set[str],
     #  character. This option is allowed only when using CSV format.
     #
 
-    if not hints['doublequote']:
+    if not hints.doublequote:
         cant_handle_hint(fail_if_cant_handle_hint, 'doublequote', hints)
     else:
         quiet_remove(unhandled_hints, 'doublequote')
 
-    if hints['escape'] is not None:
+    if hints.escape is not None:
         cant_handle_hint(fail_if_cant_handle_hint, 'escape', hints)
     else:
         quiet_remove(unhandled_hints, 'escape')
@@ -71,7 +72,7 @@ def postgres_copy_options_csv(unhandled_hints: Set[str],
     #  option is allowed only in COPY TO, and only when using CSV
     #  format.
     if mode is CopyOptionsMode.LOADING:
-        if hints['quoting'] != 'minimal':
+        if hints.quoting != 'minimal':
             cant_handle_hint(fail_if_cant_handle_hint, 'quoting', hints)
         else:
             quiet_remove(unhandled_hints, 'quoting')
@@ -86,9 +87,9 @@ def postgres_copy_options_csv(unhandled_hints: Set[str],
         # FORCE_QUOTE to force quotes when outputting non-NULL values
         # in specific columns.
 
-        if hints['quoting'] == 'minimal':
+        if hints.quoting == 'minimal':
             pass  # default
-        elif hints['quoting'] == 'all':
+        elif hints.quoting == 'all':
             postgres_options['force_quote'] = '*'
         else:
             cant_handle_hint(fail_if_cant_handle_hint, 'quoting', hints)
@@ -126,21 +127,21 @@ def postgres_copy_options_csv(unhandled_hints: Set[str],
     # dockerized-postgres public bigqueryformat # loads fine
 
     if mode is CopyOptionsMode.LOADING:
-        if hints['record-terminator'] in ("\n", "\r\n", "\r", None):
+        if hints.record_terminator in ("\n", "\r\n", "\r", None):
             quiet_remove(unhandled_hints, 'record-terminator')
         else:
             cant_handle_hint(fail_if_cant_handle_hint, 'records-terminator', hints)
     elif mode is CopyOptionsMode.UNLOADING:
         # No control for this is given - exports appear with unix
         # newlines.
-        if hints['record-terminator'] == "\n":
+        if hints.record_terminator == "\n":
             quiet_remove(unhandled_hints, 'record-terminator')
         else:
             cant_handle_hint(fail_if_cant_handle_hint, 'records-terminator', hints)
     else:
         _assert_never(mode)
 
-    if hints['compression'] is not None:
+    if hints.compression is not None:
         cant_handle_hint(fail_if_cant_handle_hint, 'compression', hints)
     else:
         quiet_remove(unhandled_hints, 'compression')

records_mover/db/postgres/copy_options/date_input_style.py

@@ -1,11 +1,11 @@
 from records_mover.utils import quiet_remove
-from records_mover.records.delimited import cant_handle_hint, RecordsHints
+from records_mover.records.delimited import cant_handle_hint, ValidatedRecordsHints
 from typing import Optional, Set
 from .types import DateOrderStyle
 
 
 def determine_input_date_order_style(unhandled_hints: Set[str],
-                                     hints: RecordsHints,
+                                     hints: ValidatedRecordsHints,
                                      fail_if_cant_handle_hint: bool) ->\
         Optional[DateOrderStyle]:
     date_order_style: Optional[DateOrderStyle] = None
@@ -39,7 +39,7 @@ def upgrade_date_order_style(style: DateOrderStyle, hint_name: str) -> None:
     #
     # postgres=#
 
-    datetimeformattz = hints['datetimeformattz']
+    datetimeformattz = hints.datetimeformattz
 
     # datetimeformattz: Valid values: "YYYY-MM-DD HH:MI:SSOF",
     # "YYYY-MM-DD HH:MI:SS", "YYYY-MM-DD HH24:MI:SSOF", "YYYY-MM-DD
@@ -94,7 +94,7 @@ def upgrade_date_order_style(style: DateOrderStyle, hint_name: str) -> None:
     # "YYYY-MM-DD HH12:MI AM", "MM/DD/YY HH24:MI". See Redshift docs
     # for more information.
 
-    datetimeformat = hints['datetimeformat']
+    datetimeformat = hints.datetimeformat
 
     if datetimeformat in ("YYYY-MM-DD HH24:MI:SS",
                           "YYYY-MM-DD HH:MI:SS"):
@@ -135,7 +135,7 @@ def upgrade_date_order_style(style: DateOrderStyle, hint_name: str) -> None:
     else:
         cant_handle_hint(fail_if_cant_handle_hint, 'datetimeformat', hints)
 
-    timeonlyformat = hints['timeonlyformat']
+    timeonlyformat = hints.timeonlyformat
 
     # timeonlyformat: Valid values: "HH12:MI AM" (e.g., "1:00 PM"),
     # "HH24:MI:SS" (e.g., "13:00:00")
@@ -171,7 +171,7 @@ def upgrade_date_order_style(style: DateOrderStyle, hint_name: str) -> None:
         cant_handle_hint(fail_if_cant_handle_hint, 'datetimeformat', hints)
 
     # dateformat: Valid values: null, "YYYY-MM-DD", "MM-DD-YYYY", "DD-MM-YYYY", "MM/DD/YY".
-    dateformat = hints['dateformat']
+    dateformat = hints.dateformat
 
     if dateformat == "YYYY-MM-DD":
         #  postgres=# select date '1999-01-02';

records_mover/db/postgres/copy_options/date_output_style.py

@@ -1,20 +1,20 @@
 from records_mover.utils import quiet_remove
-from records_mover.records.delimited import cant_handle_hint, RecordsHints
+from records_mover.records.delimited import cant_handle_hint, ValidatedRecordsHints
 from typing import Set, Tuple, Optional
 from .types import DateOrderStyle, DateOutputStyle
 
 
 def determine_date_output_style(unhandled_hints: Set[str],
-                                hints: RecordsHints,
+                                hints: ValidatedRecordsHints,
                                 fail_if_cant_handle_hint: bool) -> \
         Tuple[DateOutputStyle, Optional[DateOrderStyle]]:
 
     # see docs in the types module
 
-    dateformat = hints['dateformat']
-    timeonlyformat = hints['timeonlyformat']
-    datetimeformattz = hints['datetimeformattz']
-    datetimeformat = hints['datetimeformat']
+    dateformat = hints.dateformat
+    timeonlyformat = hints.timeonlyformat
+    datetimeformattz = hints.datetimeformattz
+    datetimeformat = hints.datetimeformat
 
     date_order_style: Optional[DateOrderStyle] = None
 

records_mover/db/postgres/copy_options/text.py

@@ -1,13 +1,14 @@
 from records_mover.utils import quiet_remove
-from records_mover.records.delimited import cant_handle_hint, RecordsHints
+from records_mover.records.records_format import DelimitedRecordsFormat
+from records_mover.records.delimited import cant_handle_hint, ValidatedRecordsHints
 from typing import Set
 from .mode import CopyOptionsMode
 from .types import PostgresCopyOptions, CopyOptionsModeType, _assert_never
 from .common import postgres_copy_options_common
 
 
 def postgres_copy_options_text(unhandled_hints: Set[str],
-                               hints: RecordsHints,
+                               hints: ValidatedRecordsHints,
                                fail_if_cant_handle_hint: bool,
                                mode: CopyOptionsModeType) ->\
         PostgresCopyOptions:
@@ -26,7 +27,7 @@ def postgres_copy_options_text(unhandled_hints: Set[str],
     # preceded by a backslash if they appear as part of a column
     # value: backslash itself, newline, carriage return, and the
     # current delimiter character.
-    if hints['escape'] is None:
+    if hints.escape is None:
         cant_handle_hint(fail_if_cant_handle_hint, 'escape', hints)
     else:
         quiet_remove(unhandled_hints, 'escape')
@@ -64,7 +65,7 @@ def postgres_copy_options_text(unhandled_hints: Set[str],
     #  character. This option is allowed only when using CSV format.
     #
 
-    if hints['doublequote']:
+    if hints.doublequote:
         cant_handle_hint(fail_if_cant_handle_hint, 'doublequote', hints)
     else:
         quiet_remove(unhandled_hints, 'doublequote')
@@ -78,7 +79,7 @@ def postgres_copy_options_text(unhandled_hints: Set[str],
     #  format.
     #
 
-    if hints['quoting'] is not None:
+    if hints.quoting is not None:
         cant_handle_hint(fail_if_cant_handle_hint, 'quoting', hints)
     else:
         quiet_remove(unhandled_hints, 'quoting')
@@ -105,21 +106,21 @@ def postgres_copy_options_text(unhandled_hints: Set[str],
     # are not all alike.
 
     if mode is CopyOptionsMode.LOADING:
-        if hints['record-terminator'] in ["\n", "\r", "\r\n"]:
+        if hints.record_terminator in ["\n", "\r", "\r\n"]:
             quiet_remove(unhandled_hints, 'record-terminator')
         else:
             cant_handle_hint(fail_if_cant_handle_hint, 'record-terminator', hints)
     elif mode is CopyOptionsMode.UNLOADING:
         # No control for this is given - exports appear with unix
         # newlines.
-        if hints['record-terminator'] == "\n":
+        if hints.record_terminator == "\n":
             quiet_remove(unhandled_hints, 'record-terminator')
         else:
             cant_handle_hint(fail_if_cant_handle_hint, 'record-terminator', hints)
     else:
         _assert_never(mode)
 
-    if hints['compression'] is not None:
+    if hints.compression is not None:
         cant_handle_hint(fail_if_cant_handle_hint, 'compression', hints)
     else:
         quiet_remove(unhandled_hints, 'compression')

records_mover/db/redshift/loader.py

@@ -49,7 +49,10 @@ def load(self,
         to = Table(table, self.meta, schema=schema)  # no autoload
         unhandled_hints = set(load_plan.records_format.hints.keys())
         processing_instructions = load_plan.processing_instructions
-        redshift_options = redshift_copy_options(unhandled_hints, load_plan.records_format.hints,
+        validated_hints = load_plan.records_format.\
+            validate(fail_if_cant_handle_hint=processing_instructions.fail_if_cant_handle_hint)
+        redshift_options = redshift_copy_options(unhandled_hints,
+                                                 validated_hints,
                                                  processing_instructions.fail_if_cant_handle_hint,
                                                  processing_instructions.fail_if_row_invalid,
                                                  processing_instructions.max_failure_rows)
@@ -106,7 +109,10 @@ def can_load_this_format(self, source_records_format: BaseRecordsFormat) -> bool
                 return False
             unhandled_hints = set(load_plan.records_format.hints.keys())
             processing_instructions = load_plan.processing_instructions
-            redshift_copy_options(unhandled_hints, load_plan.records_format.hints,
+            hints = load_plan.records_format.\
+                validate(fail_if_cant_handle_hint=processing_instructions.fail_if_cant_handle_hint)
+            redshift_copy_options(unhandled_hints,
+                                  hints,
                                   processing_instructions.fail_if_cant_handle_hint,
                                   processing_instructions.fail_if_row_invalid,
                                   processing_instructions.max_failure_rows)