RM-33-migrate-from-nose-to-pytest

.circleci/config.yml

@@ -29,9 +29,11 @@ commands:
         default: ""
     steps:
       - restore_cache:
-          key: deps-v5-<<parameters.python_version>>-<<parameters.pandas_version>>-<<parameters.extras>>-<<parameters.include_dev_dependencies>>-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "setup.py" }}
+          key: deps-v6-<<parameters.python_version>>-<<parameters.pandas_version>>-<<parameters.extras>>-<<parameters.include_dev_dependencies>>-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "setup.py" }}
       - run:
           name: Install python deps in venv
+          environment:
+            PYENV_VERSION: <<parameters.python_version>>
           command: |
             if [ -f venv/bin/activate ]
             then
@@ -60,7 +62,7 @@ commands:
               fi
             fi
       - save_cache:
-          key: deps-v5-<<parameters.python_version>>-<<parameters.pandas_version>>-<<parameters.extras>>-<<parameters.include_dev_dependencies>>-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "setup.py" }}
+          key: deps-v6-<<parameters.python_version>>-<<parameters.pandas_version>>-<<parameters.extras>>-<<parameters.include_dev_dependencies>>-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "setup.py" }}
           paths:
             - "venv"
   wait_for_db:
@@ -108,7 +110,7 @@ jobs:
         default: false
         description: "Enforce coverage not slipping"
     docker:
-      - image: circleci/python:<<parameters.python_version>>
+      - image: cimg/python:<<parameters.python_version>>
     steps:
       - checkout
       - add_ssh_keys:
@@ -176,7 +178,7 @@ jobs:
         type: string
         description: "Command to run in records-mover venv"
     docker:
-      - image: circleci/python:<<parameters.python_version>>
+      - image: cimg/python:<<parameters.python_version>>
       - image: jbfavre/vertica:8.1.1-16_centos-7
         environment:
           DATABASE_NAME: docker
@@ -280,7 +282,7 @@ jobs:
         default: true
 
     docker:
-      - image: circleci/python:<<parameters.python_version>>
+      - image: cimg/python:<<parameters.python_version>>
     steps:
       - checkout
       - add_ssh_keys:
@@ -311,7 +313,7 @@ jobs:
               # This is set by default in the CircleCI environment
               unset SCRATCH_GCS_URL
             fi
-            with-db <<parameters.db_name>> nosetests --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml .
+            with-db <<parameters.db_name>> pytest
       - store_test_results:
           path: test-reports
       - store_artifacts:
@@ -328,7 +330,7 @@ jobs:
         description: "Version of python to test against"
         default: '3.9'
     docker:
-      - image: circleci/python:<<parameters.python_version>>
+      - image: cimg/python:<<parameters.python_version>>
     steps:
       - checkout
       - installvenv:
@@ -366,7 +368,7 @@ jobs:
             twine upload -r pypi dist/*
   cli-extra-test:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.9
     steps:
       - checkout
       - installvenv:
@@ -443,7 +445,7 @@ workflows:
       #       # scratch bucket config.
       #       #
       #       unset SCRATCH_S3_URL AWS_SESSION_TOKEN AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID
-      #       with-db dockerized-vertica nosetests --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml .
+      #       with-db dockerized-vertica pytest
       #     requires:
       #       - redshift-s3-itest
       #     filters:
@@ -461,7 +463,7 @@ workflows:
             export RECORDS_MOVER_SESSION_TYPE=env
             mkdir -p test-reports/itest
             cd tests/integration/records/single_db
-            with-db dockerized-postgres nosetests --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml .
+            with-db dockerized-postgres pytest
           requires:
             - redshift-s3-itest
           filters:
@@ -483,7 +485,7 @@ workflows:
       #       export RECORDS_MOVER_SESSION_TYPE=env
       #       mkdir -p test-reports/itest
       #       cd tests/integration/records/single_db
-      #       with-db dockerized-mysql nosetests --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml .
+      #       with-db dockerized-mysql pytest
       #     requires:
       #       - redshift-s3-itest
       #     filters:
@@ -500,7 +502,7 @@ workflows:
             export RECORDS_MOVER_SESSION_TYPE=env
             mkdir -p test-reports/itest
             cd tests/integration/records/single_db
-            with-db dockerized-vertica nosetests --with-xunit --xunit-file=../../../../test-reports/itest/junit.xml .
+            with-db dockerized-vertica pytest
           requires:
             - redshift-s3-itest
           filters:

.gitignore

@@ -39,7 +39,7 @@ htmlcov/
 .tox/
 .coverage*
 .cache
-nosetests.xml
+pytest.xml
 coverage.xml
 *,cover
 /test-reports/
@@ -57,12 +57,12 @@ docs/_build/
 # PyBuilder
 target/
 
-# nosetest
+# pytesttest
 /cover
 /typecover
 /.coverage
 /coverage.xml
-/nosetests.xml
+/pytest.xml
 /test-reports
 
 .DS_Store

Makefile

@@ -25,28 +25,28 @@ citypecoverage: typecoverage
 	@test -z "$$(git status --porcelain metrics/mypy_high_water_mark)"
 
 unit:
-	ENV=test nosetests --cover-package=records_mover --cover-erase --with-coverage --with-xunit --cover-html --cover-xml --cover-inclusive tests/unit
+	ENV=test pytest --cov=records_mover tests/unit
 	mv .coverage .coverage-unit
 
 component:
-	ENV=test nosetests --cover-package=records_mover --with-coverage --with-xunit --cover-html --cover-xml --cover-inclusive tests/component
+	ENV=test pytest --cov=records_mover tests/component
 	mv .coverage .coverage-component
 
 test: unit component
-	coverage combine .coverage-unit .coverage-component # https://stackoverflow.com/questions/7352319/nosetests-combined-coverage
+	coverage combine .coverage-unit .coverage-component # https://stackoverflow.com/questions/7352319/pytest-combined-coverage
 	coverage html --directory=cover
 	coverage xml
 
 ciunit:
-	ENV=test nosetests --cover-package=records_mover --cover-erase --with-coverage --with-xunit --cover-html --cover-xml --cover-inclusive --xunit-file=test-reports/junit.xml tests/unit
+	ENV=test pytest --cov=records_mover tests/unit
 	mv .coverage .coverage-unit
 
 cicomponent:
-	ENV=test nosetests --cover-package=records_mover --with-coverage --with-xunit --cover-html --cover-xml --cover-inclusive --xunit-file=test-reports/junit.xml tests/component
+	ENV=test pytest --cov=records_mover tests/component
 	mv .coverage .coverage-component
 
 citest: test-reports ciunit cicomponent
-	coverage combine .coverage-unit .coverage-component # https://stackoverflow.com/questions/7352319/nosetests-combined-coverage
+	coverage combine .coverage-unit .coverage-component # https://stackoverflow.com/questions/7352319/pytest-combined-coverage
 	coverage html --directory=cover
 	coverage xml
 

metrics/bigfiles_high_water_mark

@@ -1 +1 @@
-1137
+1138

metrics/coverage_high_water_mark

@@ -1 +1 @@
-93.6200
+93.0000

setup.cfg

@@ -80,7 +80,7 @@ ignore_missing_imports = True
 [mypy-pyarrow.*]
 ignore_missing_imports = True
 
-[mypy-nose.*]
+[mypy-pytest.*]
 ignore_missing_imports = True
 
 [mypy-airflow.hooks.*]

setup.py

@@ -120,16 +120,17 @@ def initialize_options(self) -> None:
     'grpcio<2.0dev,>=1.29.0',
 ]
 
-nose_dependencies = [
-    'nose'
+pytest_dependencies = [
+    'pytest',
+    'pytest-cov'
 ]
 
 itest_dependencies = [
     'jsonschema',  # needed for directory_validator.py
     'pytz',
     'wheel',  # needed to support legacy 'setup.py install'
 ] + (
-    nose_dependencies +
+    pytest_dependencies +
     # needed for records_database_fixture retrying drop/creates on
     # BigQuery
     google_api_client_dependencies
@@ -255,7 +256,7 @@ def initialize_options(self) -> None:
     'coverage',
     'mock',
 ] + (
-    nose_dependencies +
+    pytest_dependencies +
     cli_dependencies_base +
     airflow_dependencies +
     gsheet_dependencies +

tests/component/records/schema/field/test_dtype.py

@@ -1,4 +1,3 @@
-from nose.tools import assert_equal
 from mock import patch
 from records_mover.records.schema.field import RecordsSchemaField
 from records_mover.records.schema.field.constraints import (
@@ -29,7 +28,7 @@ def check_dtype(field_type, constraints, expectation):
         representations=None,
     )
     out = field.cast_series_type(pd.Series(1, dtype=np.int8))
-    assert_equal(out.dtype, expectation)
+    assert out.dtype == expectation
 
 
 def test_to_pandas_dtype_integer_no_nullable():

tests/integration/itest

@@ -210,91 +210,91 @@ def run_test(args, target, parser):
         with dockerized_dbs():
             if (args.docker):
                 docker_compose_run(['with-db', 'dockerized-vertica',
-                                    'nosetests', '--xunit-file=nosetests.xml', '.'],
+                                    'pytest', '--xunit-file=pytest.xml', '.'],
                                    cwd="/usr/src/app/tests/integration/records/single_db")
 
             else:
                 with local_dockerized_dbfacts():
                     subprocess.check_call(['with-db', 'dockerized-vertica',
-                                           'nosetests', '--xunit-file=nosetests.xml', '.'],
+                                           'pytest', '--xunit-file=pytest.xml', '.'],
                                           cwd=f"{script_dir}/records/single_db")
     elif (target == 'vertica-s3'):
         with dockerized_dbs(), set_s3_scratch_bucket():
             if (args.docker):
                 docker_compose_run(['with-db', 'dockerized-vertica',
-                                    'nosetests', '--xunit-file=nosetests.xml', '.'],
+                                    'pytest', '--xunit-file=pytest.xml', '.'],
                                    prefixes=["with-aws-creds", "circleci"],
                                    cwd="/usr/src/app/tests/integration/records/single_db")
             else:
                 with local_dockerized_dbfacts():
                     subprocess.check_call(["with-db", "dockerized-vertica",
                                            "with-aws-creds", "circleci",
-                                           "nosetests", "--xunit-file=nosetests.xml", "."],
+                                           "pytest", "--xunit-file=pytest.xml", "."],
                                           cwd=f"{script_dir}/records/single_db")
     elif (target == 'mysql'):
         with dockerized_dbs():
             if (args.docker):
                 docker_compose_run(['with-db', 'dockerized-mysql',
-                                    'nosetests', '--xunit-file=nosetests.xml', '.'],
+                                    'pytest', '--xunit-file=pytest.xml', '.'],
                                    prefixes=["with-aws-creds", "circleci"],
                                    cwd="/usr/src/app/tests/integration/records/single_db")
             else:
                 with local_dockerized_dbfacts():
                     subprocess.check_call(["with-db", "dockerized-mysql",
-                                           "nosetests", "--xunit-file=nosetests.xml", "."],
+                                           "pytest", "--xunit-file=pytest.xml", "."],
                                           cwd=f"{script_dir}/records/single_db")
     elif (target == 'postgres'):
         with dockerized_dbs():
             if (args.docker):
                 docker_compose_run(['with-db', 'dockerized-postgres',
-                                    'nosetests', '--xunit-file=nosetests.xml', '.'],
+                                    'pytest', '--xunit-file=pytest.xml', '.'],
                                    prefixes=["with-aws-creds", "circleci"],
                                    cwd="/usr/src/app/tests/integration/records/single_db")
             else:
                 with local_dockerized_dbfacts():
                     subprocess.check_call(["with-db", "dockerized-postgres",
-                                           "nosetests", "--xunit-file=nosetests.xml", "."],
+                                           "pytest", "--xunit-file=pytest.xml", "."],
                                           cwd=f"{script_dir}/records/single_db")
     elif (target == 'redshift-s3'):
         with set_s3_scratch_bucket():
             if (args.docker):
                 docker_compose_run(['with-db', 'demo-itest',
-                                    'nosetests', '--xunit-file=nosetests.xml', '.'],
+                                    'pytest', '--xunit-file=pytest.xml', '.'],
                                    prefixes=["with-aws-creds", "circleci"],
                                    cwd="/usr/src/app/tests/integration/records/single_db")
             else:
                 subprocess.check_call(['with-db', 'demo-itest',
                                        "with-aws-creds", "circleci",
-                                       'nosetests', '--xunit-file=nosetests.xml', '.'],
+                                       'pytest', '--xunit-file=pytest.xml', '.'],
                                       cwd=f"{script_dir}/records/single_db")
     elif (target == 'redshift-no-s3'):
         if (args.docker):
             docker_compose_run(['with-db', 'demo-itest',
-                                'nosetests', '--xunit-file=nosetests.xml', '.'],
+                                'pytest', '--xunit-file=pytest.xml', '.'],
                                prefixes=["with-aws-creds", "circleci"],
                                cwd="/usr/src/app/tests/integration/records/single_db")
         else:
             subprocess.check_call(['with-db', 'demo-itest',
-                                   'nosetests', '--xunit-file=nosetests.xml', '.'],
+                                   'pytest', '--xunit-file=pytest.xml', '.'],
                                   cwd=f"{script_dir}/records/single_db")
     elif (target == 'bigquery-no-gcs'):
         if (args.docker):
             docker_compose_run(['with-db', 'bltoolsdevbq-bq_itest',
-                                'nosetests', '--xunit-file=nosetests.xml', '.'],
+                                'pytest', '--xunit-file=pytest.xml', '.'],
                                cwd="/usr/src/app/tests/integration/records/single_db")
         else:
             subprocess.check_call(['with-db', 'bltoolsdevbq-bq_itest',
-                                   'nosetests', '--xunit-file=nosetests.xml', '.'],
+                                   'pytest', '--xunit-file=pytest.xml', '.'],
                                   cwd=f"{script_dir}/records/single_db")
     elif (target == 'bigquery-gcs'):
         with set_gcs_scratch_bucket():
             if (args.docker):
                 docker_compose_run(['with-db', 'bltoolsdevbq-bq_itest',
-                                    'nosetests', '--xunit-file=nosetests.xml', '.'],
+                                    'pytest', '--xunit-file=pytest.xml', '.'],
                                    cwd="/usr/src/app/tests/integration/records/single_db")
             else:
                 subprocess.check_call(['with-db', 'bltoolsdevbq-bq_itest',
-                                       'nosetests', '--xunit-file=nosetests.xml', '.'],
+                                       'pytest', '--xunit-file=pytest.xml', '.'],
                                       cwd=f"{script_dir}/records/single_db")
     elif (target == 'table2table'):
         with set_s3_scratch_bucket(), set_gcs_scratch_bucket(), dockerized_dbs():