diff --git a/metrics/coverage_high_water_mark b/metrics/coverage_high_water_mark index 5f2eabeef..4c143520f 100644 --- a/metrics/coverage_high_water_mark +++ b/metrics/coverage_high_water_mark @@ -1 +1 @@ -93.6700 +93.1600 diff --git a/tests/component/db/vertica/test_vertica_import_options.py b/tests/component/db/vertica/test_vertica_import_options.py index 56b5a1a62..5ddb6a55c 100644 --- a/tests/component/db/vertica/test_vertica_import_options.py +++ b/tests/component/db/vertica/test_vertica_import_options.py @@ -2,10 +2,7 @@ from records_mover.records.load_plan import RecordsLoadPlan from records_mover.records.processing_instructions import ProcessingInstructions from records_mover.db.vertica.records_import_options import vertica_import_options -from ...records.format_hints import christmas_tree_format_1_hints -from records_mover.records.delimited.utils import logger as driver_logger import unittest -from mock import call, patch class TestVerticaImportOptions(unittest.TestCase): @@ -87,34 +84,6 @@ def test_vertica_format_permissive(self): self.assertDictEqual(options, expected_options) self.assertEqual(unhandled_hints, set()) - def test_christmas_tree_format_1_permissive(self): - vertica_format = DelimitedRecordsFormat(variant='dumb', hints=christmas_tree_format_1_hints) - processing_instructions = ProcessingInstructions(fail_if_cant_handle_hint=False) - load_plan = RecordsLoadPlan(processing_instructions=processing_instructions, - records_format=vertica_format) - unhandled_hints = set(load_plan.records_format.hints.keys()) - with patch.object(driver_logger, 'warning') as mock_warning: - options = vertica_import_options(unhandled_hints, load_plan) - expected_options = { - 'abort_on_error': True, - 'delimiter': '\x01', - 'enforcelength': True, - 'error_tolerance': False, - 'escape_as': '\\', - 'load_method': 'AUTO', - 'no_commit': False, - 'null_as': None, - 'record_terminator': '\x02', - 'rejectmax': 1, - 'skip': 1, - 'trailing_nullcols': False, - } - self.assertDictEqual(options, expected_options) - self.assertListEqual(mock_warning.mock_calls, - [call("Ignoring hint compression = 'LZO'"), - call("Ignoring hint quoting = 'nonnumeric'")]) - self.assertEqual(unhandled_hints, set()) - def test_weird_timeonlyformat(self): vertica_format = DelimitedRecordsFormat(variant='dumb', hints={ 'timeonlyformat': 'something else' diff --git a/tests/component/records/format_hints.py b/tests/component/records/format_hints.py index 6c8ac69b0..ae317450c 100644 --- a/tests/component/records/format_hints.py +++ b/tests/component/records/format_hints.py @@ -45,67 +45,3 @@ 'datetimeformattz': 'YYYY-MM-DD HH:MI:SSOF', 'header-row': False, } - -christmas_tree_format_1_hints = { - 'field-delimiter': '\001', - 'record-terminator': '\002', - 'compression': 'LZO', - 'quoting': 'nonnumeric', - 'quotechar': '"', - 'doublequote': False, - 'escape': '\\', - 'encoding': 'UTF8', - 'dateformat': 'YYYY-MM-DD', - 'timeonlyformat': 'HH24:MI:SS', - 'datetimeformat': 'YYYY-MM-DD HH24:MI:SS', - 'datetimeformattz': 'YYYY-MM-DD HH:MI:SSOF', - 'header-row': True, -} - -christmas_tree_format_2_hints = { - 'field-delimiter': '\001', - 'record-terminator': '\002', - 'compression': 'BZIP', - 'quoting': 'all', - 'quotechar': '"', - 'doublequote': True, - 'escape': '@', # not really allowed in the spec, but let's see what happens - 'encoding': 'UTF8', - 'dateformat': 'MM-DD-YYYY', - 'timeonlyformat': 'HH24:MI:SS', - 'datetimeformat': 'YYYY-MM-DD HH24:MI:SS', - 'datetimeformattz': 'HH:MI:SSOF YYYY-MM-DD', # also not allowed - 'header-row': False, -} - -christmas_tree_format_3_hints = { - 'field-delimiter': '\001', - 'record-terminator': '\002', - 'compression': 'BZIP', - 'quoting': 'some_future_option_not_supported_now', - 'quotechar': '"', - 'doublequote': True, - 'escape': '@', # not really allowed in the spec, but let's see what happens - 'encoding': 'UTF8', - 'dateformat': 'DD-MM-YYYY', - 'timeonlyformat': 'HH24:MI:SS', - 'datetimeformat': 'YYYY-MM-DD HH24:MI:SS', - 'datetimeformattz': 'HH:MI:SSOF YYYY-MM-DD', # also not allowed - 'header-row': False, -} - -christmas_tree_format_4_hints = { - 'field-delimiter': '\001', - 'record-terminator': '\002', - 'compression': 'BZIP', - 'quoting': 'some_future_option_not_supported_now', - 'quotechar': '"', - 'doublequote': True, - 'escape': '@', # not really allowed in the spec, but let's see what happens - 'encoding': 'UTF8', - 'dateformat': 'totally_bogus_just_made_this_up', - 'timeonlyformat': 'HH24:MI:SS', - 'datetimeformat': 'YYYY-MM-DD HH24:MI:SS', - 'datetimeformattz': 'HH:MI:SSOF YYYY-MM-DD', # also not allowed - 'header-row': False, -} diff --git a/tests/unit/db/redshift/test_redshift_db_driver_import_christmas_tree.py b/tests/unit/db/redshift/test_redshift_db_driver_import_christmas_tree.py deleted file mode 100644 index 9a4039341..000000000 --- a/tests/unit/db/redshift/test_redshift_db_driver_import_christmas_tree.py +++ /dev/null @@ -1,81 +0,0 @@ -from .base_test_redshift_db_driver import BaseTestRedshiftDBDriver -from records_mover.db.redshift.redshift_db_driver import Table -from mock import call, patch -from records_mover.records.delimited.utils import logger as driver_logger -from ...records.format_hints import (christmas_tree_format_1_hints, - christmas_tree_format_2_hints) -from sqlalchemy_redshift.commands import Encoding, Compression - - -class TestRedshiftDBDriverImportBlueLabs(BaseTestRedshiftDBDriver): - maxDiff = None - - @patch('records_mover.db.redshift.loader.CopyCommand') - def test_load_vertica_christmas_tree_unsupported_options_with_fast_warns_1(self, - mock_CopyCommand): - with patch.object(driver_logger, 'warning') as mock_warning: - - lines_scanned = self.load(christmas_tree_format_1_hints, fail_if=False) - - self.assertListEqual(mock_warning.mock_calls, - [call("Ignoring hint quoting = 'nonnumeric'"), - call("Ignoring hint record-terminator = '\\x02'")]) - - expected_best_effort_args = { - 'access_key_id': 'fake_aws_id', - 'compression': Compression.lzop, - 'data_location': 's3://mybucket/myparent/mychild/_manifest', - 'date_format': 'YYYY-MM-DD', - 'encoding': Encoding.utf8, - 'delimiter': '\x01', - 'escape': True, - 'ignore_header': 1, - 'manifest': True, - 'max_error': 100000, - 'quote': '"', - 'secret_access_key': 'fake_aws_secret', - 'session_token': 'fake_aws_token', - 'time_format': 'auto', - 'to': Table('mytable', self.redshift_db_driver.meta, schema='myschema'), - 'region': self.mock_directory.loc.region, - 'empty_as_null': True, - } - - mock_CopyCommand.assert_called_with(**expected_best_effort_args) - self.assertIsNone(lines_scanned) - - @patch('records_mover.db.redshift.loader.CopyCommand') - def test_load_christmas_tree_unsupported_options_with_fast_warns_2(self, - mock_CopyCommand): - with patch.object(driver_logger, 'warning') as mock_warning: - lines_scanned = self.load(christmas_tree_format_2_hints, fail_if=False) - - self.assertListEqual(mock_warning.mock_calls, - [call("Ignoring hint escape = '@'"), - call("Ignoring hint datetimeformattz = 'HH:MI:SSOF YYYY-MM-DD'"), - call("Ignoring hint doublequote = True"), - call("Ignoring hint record-terminator = '\\x02'")]) - - expected_best_effort_args = { - 'access_key_id': 'fake_aws_id', - 'compression': Compression.bzip2, - 'data_location': 's3://mybucket/myparent/mychild/_manifest', - 'date_format': 'MM-DD-YYYY', - 'delimiter': '\x01', - 'encoding': Encoding.utf8, - 'escape': True, - 'ignore_header': 0, - 'manifest': True, - 'max_error': 100000, - 'quote': '"', - 'remove_quotes': True, - 'secret_access_key': 'fake_aws_secret', - 'session_token': 'fake_aws_token', - 'time_format': 'auto', - 'to': Table('mytable', self.redshift_db_driver.meta, schema='myschema'), - 'region': self.mock_directory.loc.region, - 'empty_as_null': True, - } - - mock_CopyCommand.assert_called_with(**expected_best_effort_args) - self.assertIsNone(lines_scanned) diff --git a/tests/unit/db/redshift/test_redshift_db_driver_unload.py b/tests/unit/db/redshift/test_redshift_db_driver_unload.py index 4f0d68475..0d8d0bb79 100644 --- a/tests/unit/db/redshift/test_redshift_db_driver_unload.py +++ b/tests/unit/db/redshift/test_redshift_db_driver_unload.py @@ -1,10 +1,7 @@ from .base_test_redshift_db_driver import BaseTestRedshiftDBDriver -from ...records.format_hints import (bluelabs_format_hints, - christmas_tree_format_1_hints, - christmas_tree_format_2_hints) -from records_mover.records.delimited.utils import logger as driver_logger +from ...records.format_hints import bluelabs_format_hints from records_mover.records import DelimitedRecordsFormat -from mock import call, patch +from mock import patch def fake_text(s): @@ -85,86 +82,3 @@ def test_unload(self, } mock_UnloadFromSelect.assert_called_with(**expected_args) self.assertEqual(456, rows) - - @patch('records_mover.db.redshift.unloader.UnloadFromSelect') - @patch('records_mover.db.redshift.unloader.text') - def test_unload_christmas_tree_unsupported_options_with_fast_warns_1(self, - mock_text, - mock_UnloadFromSelect): - mock_text.side_effect = fake_text - self.mock_directory.scheme = 's3' - with patch.object(driver_logger, 'warning') as mock_warning: - self.mock_records_unload_plan.processing_instructions.fail_if_dont_understand = False - self.mock_records_unload_plan.processing_instructions.fail_if_cant_handle_hint = False - - self.mock_records_unload_plan.records_format =\ - DelimitedRecordsFormat(variant='bluelabs', - hints=christmas_tree_format_1_hints) - self.mock_db_engine.execute.return_value.scalar.return_value = 456 - rows = self.redshift_db_driver.unloader().\ - unload(schema='myschema', - table='mytable', - unload_plan=self.mock_records_unload_plan, - directory=self.mock_directory) - self.assertCountEqual(mock_warning.mock_calls, - [call("Ignoring hint record-terminator = '\\x02'"), - call("Ignoring hint quoting = 'nonnumeric'"), - call("Ignoring hint header-row = True"), - call("Ignoring hint compression = 'LZO'"), - call("Did not understand these hints: header-row=True")]) - - expected_args = { - 'access_key_id': 'fake_aws_id', - 'delimiter': '\x01', - 'escape': True, - 'manifest': True, - 'secret_access_key': 'fake_aws_secret', - 'select': ('SELECT * FROM myschema.mytable',), - 'session_token': 'fake_aws_token', - 'unload_location': 's3://mybucket/myparent/mychild/' - } - mock_UnloadFromSelect.assert_called_with(**expected_args) - self.assertEqual(456, rows) - - @patch('records_mover.db.redshift.unloader.UnloadFromSelect') - @patch('records_mover.db.redshift.unloader.text') - def test_unload_christmas_tree_unsupported_options_with_fast_warns_2(self, - mock_text, - mock_UnloadFromSelect): - mock_text.side_effect = fake_text - self.mock_directory.scheme = 's3' - with patch.object(driver_logger, 'warning') as mock_warning: - self.mock_records_unload_plan.processing_instructions.fail_if_dont_understand = False - self.mock_records_unload_plan.processing_instructions.fail_if_cant_handle_hint = False - self.mock_records_unload_plan.records_format =\ - DelimitedRecordsFormat(variant='bluelabs', - hints=christmas_tree_format_2_hints) - self.mock_db_engine.execute.return_value.scalar.return_value = 456 - rows = self.redshift_db_driver.unloader().\ - unload(schema='myschema', - table='mytable', - unload_plan=self.mock_records_unload_plan, - directory=self.mock_directory) - self.assertListEqual(mock_warning.mock_calls, - [call("Ignoring hint escape = '@'"), - call("Ignoring hint datetimeformattz = 'HH:MI:SSOF YYYY-MM-DD'"), - call("Ignoring hint record-terminator = '\\x02'"), - call("Ignoring hint doublequote = True"), - call("Ignoring hint compression = 'BZIP'"), - call("Ignoring hint datetimeformattz = " - "'YYYY-MM-DD HH24:MI:SSOF'"), - call("Ignoring hint dateformat = 'MM-DD-YYYY'")]) - - expected_args = { - 'access_key_id': 'fake_aws_id', - 'escape': True, - 'add_quotes': True, - 'delimiter': '\x01', - 'manifest': True, - 'secret_access_key': 'fake_aws_secret', - 'select': ('SELECT * FROM myschema.mytable',), - 'session_token': 'fake_aws_token', - 'unload_location': 's3://mybucket/myparent/mychild/' - } - mock_UnloadFromSelect.assert_called_with(**expected_args) - self.assertEqual(456, rows) diff --git a/tests/unit/records/test_pandas_to_csv_options_christmas_tree.py b/tests/unit/records/test_pandas_to_csv_options_christmas_tree.py deleted file mode 100644 index 770b11f03..000000000 --- a/tests/unit/records/test_pandas_to_csv_options_christmas_tree.py +++ /dev/null @@ -1,120 +0,0 @@ -from mock import patch, call -import unittest -from .format_hints import (christmas_tree_format_1_hints, christmas_tree_format_2_hints, - christmas_tree_format_3_hints, christmas_tree_format_4_hints) -from records_mover.records.delimited.utils import logger as driver_logger -from records_mover.records.pandas import pandas_to_csv_options -from records_mover.records.processing_instructions import ProcessingInstructions -from records_mover.records.records_format import DelimitedRecordsFormat - - -class TestPandasToCsvOptionsChristmasTree(unittest.TestCase): - def test_pandas_to_csv_options_christmas_tree_format_1(self): - expected = { - 'date_format': '%Y-%m-%d %H:%M:%S.%f%z', - 'doublequote': False, - 'encoding': 'UTF8', - 'escapechar': '\\', - 'header': True, - 'line_terminator': '\x02', - 'quotechar': '"', - 'quoting': 2, - 'sep': '\x01' - } - processing_instructions =\ - ProcessingInstructions(fail_if_cant_handle_hint=False) - records_format = DelimitedRecordsFormat(hints=christmas_tree_format_1_hints) - unhandled_hints = set(records_format.hints) - with patch.object(driver_logger, 'warning') as mock_warning: - actual = pandas_to_csv_options(records_format, unhandled_hints, processing_instructions) - self.assertEqual(expected, actual) - self.assertListEqual(mock_warning.mock_calls, - [call("Ignoring hint compression = 'LZO'")]) - self.assertFalse(unhandled_hints) - - def test_pandas_to_csv_options_christmas_tree_format_2(self): - expected = { - 'compression': 'bz2', - 'date_format': '%m-%d-%Y %H:%M:%S.%f%z', - 'doublequote': True, - 'encoding': 'UTF8', - 'escapechar': '\\', - 'header': False, - 'line_terminator': '\x02', - 'quotechar': '"', - 'quoting': 1, - 'sep': '\x01', - } - processing_instructions =\ - ProcessingInstructions(fail_if_cant_handle_hint=False) - records_format = DelimitedRecordsFormat(hints=christmas_tree_format_2_hints) - unhandled_hints = set(records_format.hints) - with patch.object(driver_logger, 'warning') as mock_warning: - actual = pandas_to_csv_options(records_format, unhandled_hints, processing_instructions) - self.assertEqual(expected, actual) - self.\ - assertListEqual(mock_warning.mock_calls, - [call("Ignoring hint escape = '@'"), - call("Ignoring hint datetimeformattz = 'HH:MI:SSOF YYYY-MM-DD'"), - call("Ignoring hint datetimeformattz = 'YYYY-MM-DD HH24:MI:SSOF'"), - call("Ignoring hint datetimeformat = 'YYYY-MM-DD HH24:MI:SS'")]) - self.assertFalse(unhandled_hints) - - def test_pandas_to_csv_options_christmas_tree_format_3(self): - expected = { - 'compression': 'bz2', - 'date_format': '%d-%m-%Y %H:%M:%S.%f%z', - 'doublequote': True, - 'encoding': 'UTF8', - 'escapechar': '\\', - 'header': False, - 'line_terminator': '\x02', - 'quotechar': '"', - 'quoting': 0, - 'sep': '\x01', - } - processing_instructions =\ - ProcessingInstructions(fail_if_cant_handle_hint=False) - records_format = DelimitedRecordsFormat(hints=christmas_tree_format_3_hints) - unhandled_hints = set(records_format.hints) - with patch.object(driver_logger, 'warning') as mock_warning: - actual = pandas_to_csv_options(records_format, unhandled_hints, processing_instructions) - self.assertEqual(expected, actual) - self.assertListEqual(mock_warning.mock_calls, - [call("Ignoring hint quoting = " - "'some_future_option_not_supported_now'"), - call("Ignoring hint escape = '@'"), - call("Ignoring hint datetimeformattz = 'HH:MI:SSOF YYYY-MM-DD'"), - call("Ignoring hint datetimeformattz = " - "'YYYY-MM-DD HH24:MI:SSOF'"), - call("Ignoring hint datetimeformat = 'YYYY-MM-DD HH24:MI:SS'")]) - self.assertFalse(unhandled_hints) - - def test_pandas_to_csv_options_christmas_tree_format_4(self): - expected = { - 'compression': 'bz2', - 'date_format': '%Y-%m-%d %H:%M:%S.%f%z', - 'quoting': 0, - 'doublequote': True, - 'encoding': 'UTF8', - 'escapechar': '\\', - 'header': False, - 'line_terminator': '\x02', - 'quotechar': '"', - 'sep': '\x01', - } - processing_instructions =\ - ProcessingInstructions(fail_if_cant_handle_hint=False) - records_format = DelimitedRecordsFormat(hints=christmas_tree_format_4_hints) - unhandled_hints = set(records_format.hints) - with patch.object(driver_logger, 'warning') as mock_warning: - actual = pandas_to_csv_options(records_format, unhandled_hints, processing_instructions) - self.assertEqual(expected, actual) - self.assertListEqual(mock_warning.mock_calls, - [call("Ignoring hint quoting = " - "'some_future_option_not_supported_now'"), - call("Ignoring hint escape = '@'"), - call("Ignoring hint dateformat = " - "'totally_bogus_just_made_this_up'"), - call("Ignoring hint datetimeformattz = 'HH:MI:SSOF YYYY-MM-DD'")]) - self.assertFalse(unhandled_hints)