Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

OBE: Add additional valid date formats to spec #139

Closed
wants to merge 21 commits into from
Closed

OBE: Add additional valid date formats to spec #139

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
90e3072
Add additional valid date formats to spec
vinceatbluelabs Dec 3, 2020
c0da5c2
Test some DD-MM-YYY formats
vinceatbluelabs Dec 3, 2020
d0bf905
Ensure we can close dataframe iterators once done
vinceatbluelabs Dec 3, 2020
754570a
Go straight to integration test for now
vinceatbluelabs Dec 3, 2020
22e1417
Verify we are handling all hints while processing resource_name()
vinceatbluelabs Dec 3, 2020
ea6c74e
Add type checking to integration test
vinceatbluelabs Dec 3, 2020
f916297
Restore test ordering
vinceatbluelabs Dec 3, 2020
a622047
Ensure dataframes are closed in test
vinceatbluelabs Dec 4, 2020
de78875
Temporary slip
vinceatbluelabs Dec 4, 2020
45d1f45
Fix punctuation
vinceatbluelabs Dec 4, 2020
a8edbb0
Sync mypy date/time formats with spec
vinceatbluelabs Dec 4, 2020
4edc55e
Handle dd_mm_hints
vinceatbluelabs Dec 4, 2020
493be8f
Handle dd_mm_hints
vinceatbluelabs Dec 4, 2020
ee8f4cc
Add more resources
vinceatbluelabs Dec 4, 2020
e1be3f1
Fix date formats
vinceatbluelabs Dec 4, 2020
feb7a3d
Make year format consistent
vinceatbluelabs Dec 4, 2020
37b54e2
Add more formats
vinceatbluelabs Dec 4, 2020
9163ce8
Ratchet
vinceatbluelabs Dec 4, 2020
eb5cd37
Ratchet
vinceatbluelabs Dec 4, 2020
5d41224
Add bluelabs mmdashdd files
vinceatbluelabs Dec 4, 2020
819e7d9
Add csv mmdashdd files
vinceatbluelabs Dec 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 18 additions & 9 deletions docs/RECORDS_SPEC.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ configuration provided to it outside of this spec).
Default value is `"UTF8"`

* `dateformat`: Valid values: `"YYYY-MM-DD"`, `"MM-DD-YYYY"`,
`"DD-MM-YYYY"`, `"MM/DD/YY"`.
`"DD-MM-YYYY"`, `"MM/DD/YY"`, `DD/MM/YY`.

See
[Redshift docs](https://docs.aws.amazon.com/redshift/latest/dg/r_DATEFORMAT_and_TIMEFORMAT_strings.html)
Expand All @@ -262,23 +262,32 @@ configuration provided to it outside of this spec).

Default value is `"HH24:MI"`.

* `datetimeformattz`: Valid values: `"YYYY-MM-DD HH:MI:SSOF"`,
`"YYYY-MM-DD HH:MI:SS"`, `"YYYY-MM-DD HH24:MI:SSOF"`,
and `"MM/DD/YY HH24:MI"`. See
* `datetimeformattz`: Valid values:
`"YYYY-MM-DD HH:MI:SS"`,
`"YYYY-MM-DD HH24:MI:SS"`,
`"DD-MM-YY HH24:MI"`,
`"MM-DD-YY HH24:MI"`,
`"DD/MM/YY HH24:MI"`,
`"MM/DD/YY HH24:MI"`,
as well as all the above with `OF` appended See
[Redshift docs](https://docs.aws.amazon.com/redshift/latest/dg/copy-parameters-data-conversion.html#copy-timeformat)
for more information (note that `HH:` is equivalent to `HH24:` and
that if you don't provide an offset (`OF`), times are assumed to be
in UTC).

Default value is `"YYYY-MM-DD HH:MI:SSOF"`.

* `datetimeformat`: Valid values: `"YYYY-MM-DD HH24:MI:SS"`,
`"YYYY-MM-DD HH:MI:SS"`, `"YYYY-MM-DD HH12:MI AM"`, `"MM/DD/YY HH24:MI"`.
* `datetimeformat`: Valid values:
`"YYYY-MM-DD HH:MI:SS"`,
`"YYYY-MM-DD HH12:MI AM"`.
`"YYYY-MM-DD HH24:MI:SS"`,
`"DD-MM-YY HH24:MI"`,
`"MM-DD-YY HH24:MI"`,
`"DD/MM/YY HH24:MI"`,
`"MM/DD/YY HH24:MI"`,
See
[Redshift docs](https://docs.aws.amazon.com/redshift/latest/dg/copy-parameters-data-conversion.html#copy-timeformat)
for more information (note that `HH:` is equivalent to `HH24:` and
that if you don't provide an offset (`OF`), times are assumed to be
in UTC).
for more information (note that `HH:` is equivalent to `HH24:`).

Default value is `"YYYY-MM-DD HH:MI:SS"`.

Expand Down
14 changes: 13 additions & 1 deletion records_mover/records/mover.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,19 @@ def move(records_source: RecordsSource,
logger.info(f"Mover: copying from {records_source} to {records_target} "
f"by first converting to dataframe...")
with records_source.to_dataframes_source(processing_instructions) as dataframes_source:
return move(dataframes_source, records_target, processing_instructions)
try:
return move(dataframes_source, records_target, processing_instructions)
finally:
dfs = dataframes_source.dfs
if 'close' in dir(dfs):
# Ensure database connection is closed
dfs.close() # type: ignore
else:
# TODO: Can I implement a ClosableIterator type or
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Uncompleted punchlist item detected--consider resolving or moving this to your issue tracker

# something and have mypy ensure everything
# provides this ability?
raise TypeError("Not able to close dataframes - "
"please implement a close() method on iterator")
else:
raise NotImplementedError(f"Please teach me how to move records between "
f"{records_source} and {records_target}")
24 changes: 24 additions & 0 deletions tests/integration/records/single_db/test_records_load.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,15 +34,39 @@ def load_and_verify(self, format_type, variant, hints={}, broken=False, sourcefn
def test_load_csv_format(self):
self.load_and_verify('delimited', 'csv')

def test_load_csv_format_dd_dash_mm(self) -> None:
self.load_and_verify('delimited', 'csv',
hints={
'datetimeformattz': 'DD-MM-YY HH24:MIOF',
'datetimeformat': 'DD-MM-YY HH24:MI',
'dateformat': 'DD-MM-YYYY'
})

def test_load_bigquery_format(self):
self.load_and_verify('delimited', 'bigquery')

def test_load_bigquery_format_dd_dash_mm(self):
self.load_and_verify('delimited', 'bigquery',
hints={
'datetimeformattz': 'DD-MM-YY HH24:MIOF',
'datetimeformat': 'DD-MM-YY HH24:MI',
'dateformat': 'DD-MM-YYYY'
})

def test_load_bigquery_format_with_header_row(self):
self.load_and_verify('delimited', 'bigquery', {'header-row': True})

def test_load_bluelabs_format(self):
self.load_and_verify('delimited', 'bluelabs')

def test_load_bluelabs_format_dd_dash_mm(self):
self.load_and_verify('delimited', 'bluelabs',
hints={
'datetimeformattz': 'DD-MM-YY HH24:MIOF',
'datetimeformat': 'DD-MM-YY HH24:MI',
'dateformat': 'DD-MM-YYYY'
})

def test_load_bluelabs_format_with_header_row(self):
self.load_and_verify('delimited', 'bluelabs', {'header-row': True})

Expand Down
18 changes: 18 additions & 0 deletions tests/integration/records/single_db/test_records_save_df.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,28 @@ def test_save_with_defaults(self):
hints = {}
self.save_and_verify(records_format=DelimitedRecordsFormat(hints=hints))

def test_save_with_defaults_dd_dash_mm(self):
hints = {
'datetimeformattz': 'DD-MM-YY HH24:MIOF',
'datetimeformat': 'DD-MM-YY HH24:MI',
'dateformat': 'DD-MM-YYYY'
}
self.save_and_verify(records_format=DelimitedRecordsFormat(hints=hints))

def test_save_csv_variant(self):
records_format = DelimitedRecordsFormat(variant='csv')
self.save_and_verify(records_format=records_format)

def test_save_csv_variant_dd_dash_mm(self):
hints = {
'datetimeformattz': 'DD-MM-YY HH24:MIOF',
'datetimeformat': 'DD-MM-YY HH24:MI',
'dateformat': 'DD-MM-YYYY'
}
records_format = DelimitedRecordsFormat(variant='csv',
hints=hints)
self.save_and_verify(records_format=records_format)

def test_save_with_no_compression(self):
hints = {
'compression': None,
Expand Down
32 changes: 32 additions & 0 deletions tests/integration/records/single_db/test_records_unload.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,14 @@ class RecordsUnloadIntegrationTest(BaseRecordsIntegrationTest):
def test_unload_csv_format(self):
self.unload_and_verify('delimited', 'csv')

def test_unload_csv_format_dd_dash_mm(self):
self.unload_and_verify('delimited', 'csv',
hints={
'datetimeformattz': 'DD-MM-YY HH24:MIOF',
'datetimeformat': 'DD-MM-YY HH24:MI',
'dateformat': 'DD-MM-YYYY'
})

def test_unload_csv_format_uncompressed(self):
self.unload_and_verify('delimited', 'csv', {'compression': None})

Expand All @@ -22,6 +30,14 @@ def test_unload_csv_format_without_header_row(self):
def test_unload_bluelabs_format(self):
self.unload_and_verify('delimited', 'bluelabs')

def test_unload_bluelabs_format_dd_dash_mm(self):
self.unload_and_verify('delimited', 'bluelabs',
hints={
'datetimeformattz': 'DD-MM-YY HH24:MIOF',
'datetimeformat': 'DD-MM-YY HH24:MI',
'dateformat': 'DD-MM-YYYY'
})

def test_unload_bluelabs_format_uncompressed(self):
self.unload_and_verify('delimited', 'bluelabs', {'compression': None})

Expand All @@ -31,12 +47,28 @@ def test_unload_bluelabs_format_with_header_row(self):
def test_unload_vertica_format(self):
self.unload_and_verify('delimited', 'vertica')

def test_unload_vertica_format_dd_dash_mm(self):
self.unload_and_verify('delimited', 'vertica',
hints={
'datetimeformattz': 'DD-MM-YY HH24:MIOF',
'datetimeformat': 'DD-MM-YY HH24:MI',
'dateformat': 'DD-MM-YYYY'
})

def test_unload_vertica_format_with_header_row(self):
self.unload_and_verify('delimited', 'vertica', {'header-row': True})

def test_unload_bigquery_format(self):
self.unload_and_verify('delimited', 'bigquery')

def test_unload_bigquery_format_dd_dash_mm(self):
self.unload_and_verify('delimited', 'bigquery',
hints={
'datetimeformattz': 'DD-MM-YY HH24:MIOF',
'datetimeformat': 'DD-MM-YY HH24:MI',
'dateformat': 'DD-MM-YYYY'
})

def test_unload_bigquery_format_uncompressed(self):
self.unload_and_verify('delimited', 'bigquery', {'compression': None})

Expand Down