Refactor CLI hint code and types, more CLI hint arguments

records_mover/records/delimited/hint.py

@@ -1,26 +1,37 @@
 from typing_inspect import is_literal_type, get_args
 from abc import ABCMeta, abstractmethod
 from .types import HintName, RecordsHints
+from records_mover.utils.json_schema import JsonSchemaDocument
 from typing import TypeVar, Generic, Type, List
 
 
 HintT = TypeVar('HintT')
 
 
 class Hint(Generic[HintT], metaclass=ABCMeta):
+    def __init__(self,
+                 hint_name: HintName,
+                 default: HintT,
+                 description: str) -> None:
+        self.default = default
+        self.hint_name = hint_name
+        self.description = description
+
     @abstractmethod
     def validate(self,
                  hints: RecordsHints,
                  fail_if_cant_handle_hint: bool) -> HintT:
         ...
 
+    @abstractmethod
+    def json_schema_document(self) -> JsonSchemaDocument:
+        ...
+
 
 class StringHint(Hint[str]):
-    def __init__(self,
-                 hint_name: HintName,
-                 default: str) -> None:
-        self.default = default
-        self.hint_name = hint_name
+    def json_schema_document(self) -> JsonSchemaDocument:
+        return JsonSchemaDocument('string',
+                                  description=self.description)
 
     def validate(self,
                  hints: RecordsHints,
@@ -44,12 +55,36 @@ class LiteralHint(Hint[LiteralHintT]):
     def __init__(self,
                  type_: Type[LiteralHintT],
                  hint_name: HintName,
-                 default: LiteralHintT) -> None:
+                 default: LiteralHintT,
+                 description: str) -> None:
         assert is_literal_type(type_), f"{hint_name} is not a Literal[]"
-        self.default = default
         self.type_ = type_
-        self.hint_name = hint_name
         self.valid_values: List[LiteralHintT] = list(get_args(type_))
+        super().__init__(hint_name=hint_name,
+                         default=default,
+                         description=description)
+
+    def json_schema_document(self) -> JsonSchemaDocument:
+        json_schema_types = {
+            bool: 'boolean',
+            str: 'string',
+            # Even though Python prints the word NoneType in many
+            # error messages, NoneType is not an identifier in
+            # Python. It’s not in builtins. You can only reach it with
+            # type(None).
+            #
+            # https://realpython.com/null-in-python/
+            type(None): 'null',
+        }
+
+        types_set = {
+            json_schema_types[type(valid_value)]
+            for valid_value in self.valid_values
+        }
+
+        return JsonSchemaDocument(list(types_set),
+                                  enum=self.valid_values,
+                                  description=self.description)
 
     def validate(self,
                  hints: RecordsHints,

records_mover/records/delimited/hints.py

@@ -1,16 +1,18 @@
-from .hint import LiteralHint, StringHint
+from .hint import LiteralHint, StringHint, Hint
 from .types import (
     HintHeaderRow, HintCompression, HintQuoting,
     HintDoublequote, HintEscape, HintEncoding, HintDateFormat, HintTimeOnlyFormat,
     HintDateTimeFormatTz, HintDateTimeFormat
 )
+from enum import Enum
 import logging
 
 
 logger = logging.getLogger(__name__)
 
 
-class Hints:
+class Hints(Enum):
+    value: Hint
     # mypy gives this when we pass the HintBlahBlah aliases in as an
     # argument here:
     #
@@ -19,34 +21,70 @@ class Hints:
     # Nonetheless, the validation works.
     datetimeformattz = LiteralHint[HintDateTimeFormatTz](HintDateTimeFormatTz,  # type: ignore
                                                          "datetimeformattz",
-                                                         "YYYY-MM-DD HH24:MI:SSOF")
+                                                         "YYYY-MM-DD HH24:MI:SSOF",
+                                                         description=("Format used to write "
+                                                                      "'datetimetz' values"))
     datetimeformat = LiteralHint[HintDateTimeFormat](HintDateTimeFormat,  # type: ignore
                                                      "datetimeformat",
-                                                     default="YYYY-MM-DD HH24:MI:SS")
+                                                     default="YYYY-MM-DD HH24:MI:SS",
+                                                     description=("Format used to write "
+                                                                  "'datetime' values"))
     compression = LiteralHint[HintCompression](HintCompression,  # type: ignore
                                                'compression',
-                                               default=None)
+                                               default=None,
+                                               description='Compression type of the file.')
     quoting = LiteralHint[HintQuoting](HintQuoting,  # type: ignore
                                        'quoting',
-                                       default='minimal')
+                                       default='minimal',
+                                       description=('How quotes are applied to individual fields. '
+                                                    'all: quote all fields. '
+                                                    'minimal: quote only fields that contain '
+                                                    'ambiguous characters (the '
+                                                    'delimiter, the escape character, or a line '
+                                                    'terminator). '
+                                                    'default: never quote fields.'))
     escape = LiteralHint[HintEscape](HintEscape,  # type: ignore
                                      'escape',
-                                     default='\\')
+                                     default='\\',
+                                     description="Character used to escape strings")
     encoding = LiteralHint[HintEncoding](HintEncoding,  # type: ignore
                                          'encoding',
-                                         default='UTF8')
+                                         default='UTF8',
+                                         description="Text encoding of file")
     dateformat = LiteralHint[HintDateFormat](HintDateFormat,  # type: ignore
                                              'dateformat',
-                                             default='YYYY-MM-DD')
+                                             default='YYYY-MM-DD',
+                                             description=("Format used to write "
+                                                          "'date' values"))
     timeonlyformat = LiteralHint[HintTimeOnlyFormat](HintTimeOnlyFormat,  # type: ignore
                                                      'timeonlyformat',
-                                                     default="HH24:MI:SS")
+                                                     default="HH24:MI:SS",
+                                                     description=("Format used to write "
+                                                                  "'time' values"))
+    # https://docs.python.org/3/library/csv.html#csv.Dialect.doublequote
     doublequote = LiteralHint[HintDoublequote](HintDoublequote,  # type: ignore
                                                'doublequote',
-                                               default=False)
+                                               default=False,
+                                               description=('Controls how instances of quotechar '
+                                                            'appearing inside a field should '
+                                                            'themselves be quoted. When True, the '
+                                                            'character is doubled. When False, the '
+                                                            'escapechar is used as a prefix to the '
+                                                            'quotechar.'))
     header_row = LiteralHint[HintHeaderRow](HintHeaderRow,  # type: ignore
                                             'header-row',
-                                            default=True)
-    quotechar = StringHint('quotechar', default='"')
-    record_terminator = StringHint('record-terminator', default='\n')
-    field_delimiter = StringHint('field-delimiter', default=',')
+                                            default=True,
+                                            description=('True if a header row is provided in '
+                                                         'the delimited files.'))
+    # https://docs.python.org/3/library/csv.html#csv.Dialect.quotechar
+    quotechar = StringHint('quotechar',
+                           default='"',
+                           description=('A one-character string used to quote fields containing '
+                                        'special characters, such as the delimiter or quotechar, '
+                                        'or which contain new-line characters.'))
+    record_terminator = StringHint('record-terminator',
+                                   default='\n',
+                                   description='String used to close out individual rows of data.')
+    field_delimiter = StringHint('field-delimiter',
+                                 default=',',
+                                 description='Character used between fields.')

records_mover/records/delimited/validated_records_hints.py

@@ -33,17 +33,17 @@ def v(hint: Hint[T]) -> T:
             return hint.validate(hints, fail_if_cant_handle_hint)
 
         return ValidatedRecordsHints(
-            header_row=v(Hints.header_row),
-            field_delimiter=v(Hints.field_delimiter),
-            compression=v(Hints.compression),
-            record_terminator=v(Hints.record_terminator),
-            quoting=v(Hints.quoting),
-            quotechar=v(Hints.quotechar),
-            doublequote=v(Hints.doublequote),
-            escape=v(Hints.escape),
-            encoding=v(Hints.encoding),
-            dateformat=v(Hints.dateformat),
-            timeonlyformat=v(Hints.timeonlyformat),
-            datetimeformattz=v(Hints.datetimeformattz),
-            datetimeformat=v(Hints.datetimeformat),
+            header_row=v(Hints.header_row.value),
+            field_delimiter=v(Hints.field_delimiter.value),
+            compression=v(Hints.compression.value),
+            record_terminator=v(Hints.record_terminator.value),
+            quoting=v(Hints.quoting.value),
+            quotechar=v(Hints.quotechar.value),
+            doublequote=v(Hints.doublequote.value),
+            escape=v(Hints.escape.value),
+            encoding=v(Hints.encoding.value),
+            dateformat=v(Hints.dateformat.value),
+            timeonlyformat=v(Hints.timeonlyformat.value),
+            datetimeformattz=v(Hints.datetimeformattz.value),
+            datetimeformat=v(Hints.datetimeformat.value),
         )

records_mover/records/job/hints.py

@@ -1,8 +1,15 @@
 """Defines hints supported by the job config parser."""
+from records_mover.records.delimited.hints import Hints
 from ...utils.json_schema import JsonParameter, JsonSchemaDocument
 from typing import Optional
 
 
+SUPPORTED_HINT_NAMES = [
+    'field-delimiter', 'compression', 'escape', 'quoting', 'encoding', 'header-row'
+]
+
+
+# TODO: This class isn't needed anymore if we can just provide Hint class
 class SupportedHint:
     """Definition for supported hints"""
 
@@ -17,50 +24,12 @@ def config_name(self) -> str:
         return self.schema.name
 
 
-QUOTING_DESCRIPTION =\
-        ('How quotes are applied to individual fields. '
-         'all: quote all fields. '
-         'minimal: quote only fields that contain ambiguous characters (the '
-         'delimiter, the escape character, or a line terminator). '
-         'default: never quote fields.')
-
-#
-# Note: Any expansion of these types should also be done in
-# records.types
-#
 SUPPORTED_HINTS = [
     SupportedHint(
-        JsonParameter('field-delimiter',
-                      JsonSchemaDocument('string',
-                                         description=('Character used between fields '
-                                                      '(default is comma)')),
-                      optional=True)),
-    SupportedHint(
-        JsonParameter('compression',
-                      JsonSchemaDocument(['string', 'null'],
-                                         enum=['BZIP', 'GZIP', 'LZO', None],
-                                         description='Compression type of the file.'),
-                      optional=True)),
-    SupportedHint(
-        JsonParameter('escape',
-                      JsonSchemaDocument(['string', 'null'],
-                                         enum=['\\', None],
-                                         description="Character used to escape strings"),
-                      optional=True)),
-    SupportedHint(
-        JsonParameter('quoting',
-                      JsonSchemaDocument(['string', 'null'],
-                                         enum=['all', 'minimal', 'nonnumeric', None],
-                                         description=QUOTING_DESCRIPTION),
-                      optional=True)),
-    SupportedHint(
-        JsonParameter('encoding',
-                      JsonSchemaDocument(['string'],
-                                         enum=[
-                                             'UTF8', 'UTF16', 'UTF16LE', 'UTF16BE',
-                                             'LATIN1', 'CP1252'
-                                         ],
-                                         description="Text encoding of file"),
-                      optional=True)),
+        JsonParameter(hint_enum.value.hint_name,
+                      hint_enum.value.json_schema_document(),
+                      optional=True))
+    for hint_enum in list(Hints)
 ]
+
 SUPPORTED_HINT_LOOKUP = {hint.config_name: hint for hint in SUPPORTED_HINTS}