From e2747655ba47925f8dec5f556f707d203f129071 Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Tue, 3 Jan 2023 17:01:53 -0500 Subject: [PATCH 01/21] add 3.11 test --- .circleci/config.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0c55584ea..b0c6e779a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -427,6 +427,15 @@ workflows: filters: tags: only: /v\d+\.\d+\.\d+(-[\w]+)?/ + - test: + name: test-3.11 + extras: '[unittest,typecheck]' + python_version: "3.11" + pandas_version: "==1.1.5" + coverage: true + filters: + tags: + only: /v\d+\.\d+\.\d+(-[\w]+)?/ # - integration_test_with_dbs: # name: vertica-no-s3-itest # extras: '[vertica,itest]' From c3f1d119aa3faf22246bda59d0aead3e6fd043a6 Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Tue, 3 Jan 2023 17:04:30 -0500 Subject: [PATCH 02/21] update to cimg --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b0c6e779a..8ef1146a2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -108,7 +108,7 @@ jobs: default: false description: "Enforce coverage not slipping" docker: - - image: circleci/python:<> + - image: cimg/python:<> steps: - checkout - add_ssh_keys: @@ -176,7 +176,7 @@ jobs: type: string description: "Command to run in records-mover venv" docker: - - image: circleci/python:<> + - image: cimg/python:<> - image: jbfavre/vertica:8.1.1-16_centos-7 environment: DATABASE_NAME: docker @@ -280,7 +280,7 @@ jobs: default: true docker: - - image: circleci/python:<> + - image: cimg/python:<> steps: - checkout - add_ssh_keys: @@ -328,7 +328,7 @@ jobs: description: "Version of python to test against" default: '3.9' docker: - - image: circleci/python:<> + - image: cimg/python:<> steps: - checkout - installvenv: @@ -366,7 +366,7 @@ jobs: twine upload -r pypi dist/* cli-extra-test: docker: - - image: circleci/python:3.9 + - image: cimg/python:3.9 steps: - checkout - installvenv: From 8a04b5f42202c8578466de8f97b248243ecb7586 Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Tue, 3 Jan 2023 17:23:39 -0500 Subject: [PATCH 03/21] update to v6 cache --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8ef1146a2..8310e5f1d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,7 +29,7 @@ commands: default: "" steps: - restore_cache: - key: deps-v5-<>-<>-<>-<>-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "setup.py" }} + key: deps-v6-<>-<>-<>-<>-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "setup.py" }} - run: name: Install python deps in venv command: | @@ -60,7 +60,7 @@ commands: fi fi - save_cache: - key: deps-v5-<>-<>-<>-<>-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "setup.py" }} + key: deps-v6-<>-<>-<>-<>-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "setup.py" }} paths: - "venv" wait_for_db: From 40d7d4ac74e887a1f4fea41df84c2fa285526383 Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Wed, 4 Jan 2023 12:00:41 -0500 Subject: [PATCH 04/21] set pyenv version in installenv --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8310e5f1d..ba7e07691 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,6 +32,8 @@ commands: key: deps-v6-<>-<>-<>-<>-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "setup.py" }} - run: name: Install python deps in venv + environment: + PYENV_VERSION: <> command: | if [ -f venv/bin/activate ] then From 7c511d99c0c7ffa2d2176521e4c93db61e7db0d4 Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Wed, 4 Jan 2023 12:04:46 -0500 Subject: [PATCH 05/21] update pandas version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ba7e07691..b2eed7f51 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -433,7 +433,7 @@ workflows: name: test-3.11 extras: '[unittest,typecheck]' python_version: "3.11" - pandas_version: "==1.1.5" + pandas_version: "==1.5.2" coverage: true filters: tags: From 1c57068511febf51e5b6222f94784ec1874e0311 Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Wed, 4 Jan 2023 12:09:13 -0500 Subject: [PATCH 06/21] remove versioning from pandas --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8fe949650..b3bb7cbd9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ flake8 # https://github.com/pallets/markupsafe/issues/284 markupsafe==2.0.1 # https://github.com/pandas-dev/pandas/pull/45749 -pandas==1.1.5 +pandas google-cloud-storage boto3 From 713dcf3a48cc3a5b0aabe2a124fa5e1c4f18e242 Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Wed, 4 Jan 2023 12:11:43 -0500 Subject: [PATCH 07/21] remove pandas from requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b3bb7cbd9..09fc5528a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ flake8 # https://github.com/pallets/markupsafe/issues/284 markupsafe==2.0.1 # https://github.com/pandas-dev/pandas/pull/45749 -pandas +# pandas google-cloud-storage boto3 From 363313e9d8794a0ebffdb6277969da330ff1b018 Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Wed, 4 Jan 2023 12:17:24 -0500 Subject: [PATCH 08/21] remove versioning from pandas dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7bafe58cb..1a073b669 100755 --- a/setup.py +++ b/setup.py @@ -192,7 +192,7 @@ def initialize_options(self) -> None: ] pandas_dependencies = [ - 'pandas==1.1.5', + 'pandas', ] mysql_dependencies = [ From af21e4a8b5140177a6a8d39dc73abc8d2575d8f8 Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Wed, 4 Jan 2023 12:48:05 -0500 Subject: [PATCH 09/21] remove nose --- tests/component/records/schema/field/test_dtype.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/component/records/schema/field/test_dtype.py b/tests/component/records/schema/field/test_dtype.py index 4498fe0e0..7d880a23b 100644 --- a/tests/component/records/schema/field/test_dtype.py +++ b/tests/component/records/schema/field/test_dtype.py @@ -1,4 +1,3 @@ -from nose.tools import assert_equal from mock import patch from records_mover.records.schema.field import RecordsSchemaField from records_mover.records.schema.field.constraints import ( @@ -29,7 +28,7 @@ def check_dtype(field_type, constraints, expectation): representations=None, ) out = field.cast_series_type(pd.Series(1, dtype=np.int8)) - assert_equal(out.dtype, expectation) + assert out.dtype == expectation def test_to_pandas_dtype_integer_no_nullable(): From f3526f0233a2469b8db2ecc64766012600bd444b Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Wed, 4 Jan 2023 12:49:22 -0500 Subject: [PATCH 10/21] try mass edit --- .circleci/config.yml | 10 +++++----- .gitignore | 6 +++--- Makefile | 12 ++++++------ setup.cfg | 2 +- setup.py | 8 ++++---- tests/integration/itest | 32 ++++++++++++++++---------------- 6 files changed, 35 insertions(+), 35 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b2eed7f51..d662bc4a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -313,7 +313,7 @@ jobs: # This is set by default in the CircleCI environment unset SCRATCH_GCS_URL fi - with-db <> nosetests --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml . + with-db <> pytest --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml . - store_test_results: path: test-reports - store_artifacts: @@ -454,7 +454,7 @@ workflows: # # scratch bucket config. # # # unset SCRATCH_S3_URL AWS_SESSION_TOKEN AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID - # with-db dockerized-vertica nosetests --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml . + # with-db dockerized-vertica pytest --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml . # requires: # - redshift-s3-itest # filters: @@ -472,7 +472,7 @@ workflows: export RECORDS_MOVER_SESSION_TYPE=env mkdir -p test-reports/itest cd tests/integration/records/single_db - with-db dockerized-postgres nosetests --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml . + with-db dockerized-postgres pytest --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml . requires: - redshift-s3-itest filters: @@ -494,7 +494,7 @@ workflows: # export RECORDS_MOVER_SESSION_TYPE=env # mkdir -p test-reports/itest # cd tests/integration/records/single_db - # with-db dockerized-mysql nosetests --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml . + # with-db dockerized-mysql pytest --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml . # requires: # - redshift-s3-itest # filters: @@ -511,7 +511,7 @@ workflows: export RECORDS_MOVER_SESSION_TYPE=env mkdir -p test-reports/itest cd tests/integration/records/single_db - with-db dockerized-vertica nosetests --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml . + with-db dockerized-vertica pytest --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml . requires: - redshift-s3-itest filters: diff --git a/.gitignore b/.gitignore index 8bee7430c..8b47d45a9 100644 --- a/.gitignore +++ b/.gitignore @@ -39,7 +39,7 @@ htmlcov/ .tox/ .coverage* .cache -nosetests.xml +pytest.xml coverage.xml *,cover /test-reports/ @@ -57,12 +57,12 @@ docs/_build/ # PyBuilder target/ -# nosetest +# pytesttest /cover /typecover /.coverage /coverage.xml -/nosetests.xml +/pytest.xml /test-reports .DS_Store diff --git a/Makefile b/Makefile index 5ecc0193f..04008a109 100644 --- a/Makefile +++ b/Makefile @@ -25,28 +25,28 @@ citypecoverage: typecoverage @test -z "$$(git status --porcelain metrics/mypy_high_water_mark)" unit: - ENV=test nosetests --cover-package=records_mover --cover-erase --with-coverage --with-xunit --cover-html --cover-xml --cover-inclusive tests/unit + ENV=test pytest --cover-package=records_mover --cover-erase --with-coverage --with-xunit --cover-html --cover-xml --cover-inclusive tests/unit mv .coverage .coverage-unit component: - ENV=test nosetests --cover-package=records_mover --with-coverage --with-xunit --cover-html --cover-xml --cover-inclusive tests/component + ENV=test pytest --cover-package=records_mover --with-coverage --with-xunit --cover-html --cover-xml --cover-inclusive tests/component mv .coverage .coverage-component test: unit component - coverage combine .coverage-unit .coverage-component # https://stackoverflow.com/questions/7352319/nosetests-combined-coverage + coverage combine .coverage-unit .coverage-component # https://stackoverflow.com/questions/7352319/pytest-combined-coverage coverage html --directory=cover coverage xml ciunit: - ENV=test nosetests --cover-package=records_mover --cover-erase --with-coverage --with-xunit --cover-html --cover-xml --cover-inclusive --xunit-file=test-reports/junit.xml tests/unit + ENV=test pytest --cover-package=records_mover --cover-erase --with-coverage --with-xunit --cover-html --cover-xml --cover-inclusive --xunit-file=test-reports/junit.xml tests/unit mv .coverage .coverage-unit cicomponent: - ENV=test nosetests --cover-package=records_mover --with-coverage --with-xunit --cover-html --cover-xml --cover-inclusive --xunit-file=test-reports/junit.xml tests/component + ENV=test pytest --cover-package=records_mover --with-coverage --with-xunit --cover-html --cover-xml --cover-inclusive --xunit-file=test-reports/junit.xml tests/component mv .coverage .coverage-component citest: test-reports ciunit cicomponent - coverage combine .coverage-unit .coverage-component # https://stackoverflow.com/questions/7352319/nosetests-combined-coverage + coverage combine .coverage-unit .coverage-component # https://stackoverflow.com/questions/7352319/pytest-combined-coverage coverage html --directory=cover coverage xml diff --git a/setup.cfg b/setup.cfg index 4b802d137..a620527bf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -80,7 +80,7 @@ ignore_missing_imports = True [mypy-pyarrow.*] ignore_missing_imports = True -[mypy-nose.*] +[mypy-pytest.*] ignore_missing_imports = True [mypy-airflow.hooks.*] diff --git a/setup.py b/setup.py index 1a073b669..2e4de43d0 100755 --- a/setup.py +++ b/setup.py @@ -120,8 +120,8 @@ def initialize_options(self) -> None: 'grpcio<2.0dev,>=1.29.0', ] -nose_dependencies = [ - 'nose' +pytest_dependencies = [ + 'pytest' ] itest_dependencies = [ @@ -129,7 +129,7 @@ def initialize_options(self) -> None: 'pytz', 'wheel', # needed to support legacy 'setup.py install' ] + ( - nose_dependencies + + pytest_dependencies + # needed for records_database_fixture retrying drop/creates on # BigQuery google_api_client_dependencies @@ -255,7 +255,7 @@ def initialize_options(self) -> None: 'coverage', 'mock', ] + ( - nose_dependencies + + pytest_dependencies + cli_dependencies_base + airflow_dependencies + gsheet_dependencies + diff --git a/tests/integration/itest b/tests/integration/itest index 2086cd348..353f36a77 100755 --- a/tests/integration/itest +++ b/tests/integration/itest @@ -210,91 +210,91 @@ def run_test(args, target, parser): with dockerized_dbs(): if (args.docker): docker_compose_run(['with-db', 'dockerized-vertica', - 'nosetests', '--xunit-file=nosetests.xml', '.'], + 'pytest', '--xunit-file=pytest.xml', '.'], cwd="/usr/src/app/tests/integration/records/single_db") else: with local_dockerized_dbfacts(): subprocess.check_call(['with-db', 'dockerized-vertica', - 'nosetests', '--xunit-file=nosetests.xml', '.'], + 'pytest', '--xunit-file=pytest.xml', '.'], cwd=f"{script_dir}/records/single_db") elif (target == 'vertica-s3'): with dockerized_dbs(), set_s3_scratch_bucket(): if (args.docker): docker_compose_run(['with-db', 'dockerized-vertica', - 'nosetests', '--xunit-file=nosetests.xml', '.'], + 'pytest', '--xunit-file=pytest.xml', '.'], prefixes=["with-aws-creds", "circleci"], cwd="/usr/src/app/tests/integration/records/single_db") else: with local_dockerized_dbfacts(): subprocess.check_call(["with-db", "dockerized-vertica", "with-aws-creds", "circleci", - "nosetests", "--xunit-file=nosetests.xml", "."], + "pytest", "--xunit-file=pytest.xml", "."], cwd=f"{script_dir}/records/single_db") elif (target == 'mysql'): with dockerized_dbs(): if (args.docker): docker_compose_run(['with-db', 'dockerized-mysql', - 'nosetests', '--xunit-file=nosetests.xml', '.'], + 'pytest', '--xunit-file=pytest.xml', '.'], prefixes=["with-aws-creds", "circleci"], cwd="/usr/src/app/tests/integration/records/single_db") else: with local_dockerized_dbfacts(): subprocess.check_call(["with-db", "dockerized-mysql", - "nosetests", "--xunit-file=nosetests.xml", "."], + "pytest", "--xunit-file=pytest.xml", "."], cwd=f"{script_dir}/records/single_db") elif (target == 'postgres'): with dockerized_dbs(): if (args.docker): docker_compose_run(['with-db', 'dockerized-postgres', - 'nosetests', '--xunit-file=nosetests.xml', '.'], + 'pytest', '--xunit-file=pytest.xml', '.'], prefixes=["with-aws-creds", "circleci"], cwd="/usr/src/app/tests/integration/records/single_db") else: with local_dockerized_dbfacts(): subprocess.check_call(["with-db", "dockerized-postgres", - "nosetests", "--xunit-file=nosetests.xml", "."], + "pytest", "--xunit-file=pytest.xml", "."], cwd=f"{script_dir}/records/single_db") elif (target == 'redshift-s3'): with set_s3_scratch_bucket(): if (args.docker): docker_compose_run(['with-db', 'demo-itest', - 'nosetests', '--xunit-file=nosetests.xml', '.'], + 'pytest', '--xunit-file=pytest.xml', '.'], prefixes=["with-aws-creds", "circleci"], cwd="/usr/src/app/tests/integration/records/single_db") else: subprocess.check_call(['with-db', 'demo-itest', "with-aws-creds", "circleci", - 'nosetests', '--xunit-file=nosetests.xml', '.'], + 'pytest', '--xunit-file=pytest.xml', '.'], cwd=f"{script_dir}/records/single_db") elif (target == 'redshift-no-s3'): if (args.docker): docker_compose_run(['with-db', 'demo-itest', - 'nosetests', '--xunit-file=nosetests.xml', '.'], + 'pytest', '--xunit-file=pytest.xml', '.'], prefixes=["with-aws-creds", "circleci"], cwd="/usr/src/app/tests/integration/records/single_db") else: subprocess.check_call(['with-db', 'demo-itest', - 'nosetests', '--xunit-file=nosetests.xml', '.'], + 'pytest', '--xunit-file=pytest.xml', '.'], cwd=f"{script_dir}/records/single_db") elif (target == 'bigquery-no-gcs'): if (args.docker): docker_compose_run(['with-db', 'bltoolsdevbq-bq_itest', - 'nosetests', '--xunit-file=nosetests.xml', '.'], + 'pytest', '--xunit-file=pytest.xml', '.'], cwd="/usr/src/app/tests/integration/records/single_db") else: subprocess.check_call(['with-db', 'bltoolsdevbq-bq_itest', - 'nosetests', '--xunit-file=nosetests.xml', '.'], + 'pytest', '--xunit-file=pytest.xml', '.'], cwd=f"{script_dir}/records/single_db") elif (target == 'bigquery-gcs'): with set_gcs_scratch_bucket(): if (args.docker): docker_compose_run(['with-db', 'bltoolsdevbq-bq_itest', - 'nosetests', '--xunit-file=nosetests.xml', '.'], + 'pytest', '--xunit-file=pytest.xml', '.'], cwd="/usr/src/app/tests/integration/records/single_db") else: subprocess.check_call(['with-db', 'bltoolsdevbq-bq_itest', - 'nosetests', '--xunit-file=nosetests.xml', '.'], + 'pytest', '--xunit-file=pytest.xml', '.'], cwd=f"{script_dir}/records/single_db") elif (target == 'table2table'): with set_s3_scratch_bucket(), set_gcs_scratch_bucket(), dockerized_dbs(): From 7ff0b2ee90cea3f8a996f8dad8a4ee1b0d34a1f2 Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Wed, 4 Jan 2023 13:07:25 -0500 Subject: [PATCH 11/21] update to pytest coverage idiom --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 04008a109..bf536c724 100644 --- a/Makefile +++ b/Makefile @@ -25,11 +25,11 @@ citypecoverage: typecoverage @test -z "$$(git status --porcelain metrics/mypy_high_water_mark)" unit: - ENV=test pytest --cover-package=records_mover --cover-erase --with-coverage --with-xunit --cover-html --cover-xml --cover-inclusive tests/unit + ENV=test pytest --cov=records_mover tests/unit mv .coverage .coverage-unit component: - ENV=test pytest --cover-package=records_mover --with-coverage --with-xunit --cover-html --cover-xml --cover-inclusive tests/component + ENV=test pytest --cov=records_mover tests/component mv .coverage .coverage-component test: unit component @@ -38,11 +38,11 @@ test: unit component coverage xml ciunit: - ENV=test pytest --cover-package=records_mover --cover-erase --with-coverage --with-xunit --cover-html --cover-xml --cover-inclusive --xunit-file=test-reports/junit.xml tests/unit + ENV=test pytest --cov=records_mover tests/unit mv .coverage .coverage-unit cicomponent: - ENV=test pytest --cover-package=records_mover --with-coverage --with-xunit --cover-html --cover-xml --cover-inclusive --xunit-file=test-reports/junit.xml tests/component + ENV=test pytest --cov=records_mover tests/component mv .coverage .coverage-component citest: test-reports ciunit cicomponent From 54fdfdb391b16e391ccf8135d84bc98f62db2e9f Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Wed, 4 Jan 2023 13:10:11 -0500 Subject: [PATCH 12/21] add pytest cov to test --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2e4de43d0..1769f088b 100755 --- a/setup.py +++ b/setup.py @@ -121,7 +121,8 @@ def initialize_options(self) -> None: ] pytest_dependencies = [ - 'pytest' + 'pytest', + 'pytest-cov' ] itest_dependencies = [ From 862fb033f5c5304bf72b1cabb53a9c84b79ac7a0 Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Thu, 5 Jan 2023 09:18:08 -0500 Subject: [PATCH 13/21] remove max on airflow version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1769f088b..d5766726a 100755 --- a/setup.py +++ b/setup.py @@ -139,7 +139,7 @@ def initialize_options(self) -> None: airflow_dependencies = [ # Minimum version here is needed to avoid syntax error in setup.py # in 1.10.0 - 'apache-airflow>=1.10.1,<2' + 'apache-airflow>=1.10.1' ] db_dependencies = [ From 43b7c9aff1f0e7c98677aab09e207c8319577419 Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Thu, 5 Jan 2023 09:44:50 -0500 Subject: [PATCH 14/21] Revert "remove max on airflow version" This reverts commit 862fb033f5c5304bf72b1cabb53a9c84b79ac7a0. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d5766726a..1769f088b 100755 --- a/setup.py +++ b/setup.py @@ -139,7 +139,7 @@ def initialize_options(self) -> None: airflow_dependencies = [ # Minimum version here is needed to avoid syntax error in setup.py # in 1.10.0 - 'apache-airflow>=1.10.1' + 'apache-airflow>=1.10.1,<2' ] db_dependencies = [ From 84c3c88bf7ceba2ded470ddc1e91136a94507961 Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Thu, 5 Jan 2023 09:58:56 -0500 Subject: [PATCH 15/21] remove 3.11 test --- .circleci/config.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d662bc4a9..ec26fdf11 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -429,15 +429,6 @@ workflows: filters: tags: only: /v\d+\.\d+\.\d+(-[\w]+)?/ - - test: - name: test-3.11 - extras: '[unittest,typecheck]' - python_version: "3.11" - pandas_version: "==1.5.2" - coverage: true - filters: - tags: - only: /v\d+\.\d+\.\d+(-[\w]+)?/ # - integration_test_with_dbs: # name: vertica-no-s3-itest # extras: '[vertica,itest]' @@ -680,3 +671,5 @@ workflows: only: /v\d+\.\d+\.\d+(-[\w]+)?/ branches: ignore: /.*/ + +# VS Code Extension Version: 1.5.1 \ No newline at end of file From 5ce2833e57a2e33824a86af6bc9edbf8ebbc3327 Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Thu, 5 Jan 2023 10:03:29 -0500 Subject: [PATCH 16/21] update coverage high water mark --- metrics/coverage_high_water_mark | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/coverage_high_water_mark b/metrics/coverage_high_water_mark index 9c2857e0b..d1a64d483 100644 --- a/metrics/coverage_high_water_mark +++ b/metrics/coverage_high_water_mark @@ -1 +1 @@ -93.6200 +93.0000 From 4296e6e053526c41135263989255d6b78e490871 Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Thu, 5 Jan 2023 10:04:13 -0500 Subject: [PATCH 17/21] update bigfiles high water mark --- metrics/bigfiles_high_water_mark | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/bigfiles_high_water_mark b/metrics/bigfiles_high_water_mark index 3393c6ddc..91d0f8a3f 100644 --- a/metrics/bigfiles_high_water_mark +++ b/metrics/bigfiles_high_water_mark @@ -1 +1 @@ -1137 +1138 From e34b76b66e081207f76e6672bc21c070e7d522db Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Thu, 5 Jan 2023 10:09:30 -0500 Subject: [PATCH 18/21] remove --with-xunit invocation --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ec26fdf11..45de57d91 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -313,7 +313,7 @@ jobs: # This is set by default in the CircleCI environment unset SCRATCH_GCS_URL fi - with-db <> pytest --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml . + with-db <> pytest - store_test_results: path: test-reports - store_artifacts: @@ -445,7 +445,7 @@ workflows: # # scratch bucket config. # # # unset SCRATCH_S3_URL AWS_SESSION_TOKEN AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID - # with-db dockerized-vertica pytest --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml . + # with-db dockerized-vertica pytest # requires: # - redshift-s3-itest # filters: @@ -463,7 +463,7 @@ workflows: export RECORDS_MOVER_SESSION_TYPE=env mkdir -p test-reports/itest cd tests/integration/records/single_db - with-db dockerized-postgres pytest --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml . + with-db dockerized-postgres pytest requires: - redshift-s3-itest filters: @@ -485,7 +485,7 @@ workflows: # export RECORDS_MOVER_SESSION_TYPE=env # mkdir -p test-reports/itest # cd tests/integration/records/single_db - # with-db dockerized-mysql pytest --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml . + # with-db dockerized-mysql pytest # requires: # - redshift-s3-itest # filters: @@ -502,7 +502,7 @@ workflows: export RECORDS_MOVER_SESSION_TYPE=env mkdir -p test-reports/itest cd tests/integration/records/single_db - with-db dockerized-vertica pytest --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml . + with-db dockerized-vertica pytest requires: - redshift-s3-itest filters: From 8a53e79078f75e9d759171b41b1848ccc4210cb0 Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Fri, 6 Jan 2023 08:24:50 -0500 Subject: [PATCH 19/21] remove VS code add --- .circleci/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 45de57d91..b2ff73d43 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -671,5 +671,3 @@ workflows: only: /v\d+\.\d+\.\d+(-[\w]+)?/ branches: ignore: /.*/ - -# VS Code Extension Version: 1.5.1 \ No newline at end of file From 14d427ac187869bb3c27565f83f59218886e6a9f Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Fri, 6 Jan 2023 08:26:13 -0500 Subject: [PATCH 20/21] restore pandas --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 09fc5528a..8fe949650 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ flake8 # https://github.com/pallets/markupsafe/issues/284 markupsafe==2.0.1 # https://github.com/pandas-dev/pandas/pull/45749 -# pandas +pandas==1.1.5 google-cloud-storage boto3 From ffaad2f16b03ae4cc786842bae0d9f7286d7e55e Mon Sep 17 00:00:00 2001 From: Tim Ryan Date: Fri, 6 Jan 2023 08:27:14 -0500 Subject: [PATCH 21/21] recommit pandas version number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1769f088b..9f377100b 100755 --- a/setup.py +++ b/setup.py @@ -193,7 +193,7 @@ def initialize_options(self) -> None: ] pandas_dependencies = [ - 'pandas', + 'pandas==1.1.5', ] mysql_dependencies = [