diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index c55c60e..f236632 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -15,10 +15,10 @@ jobs: with: ref: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} - - name: Set up python 3.7 + - name: Set up python 3.8 uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.8 - name: Install codecov and erase coverage run: | python -m pip install "$(cat test-requirements.txt | grep codecov)" diff --git a/.github/workflows/lint_check.yml b/.github/workflows/lint_check.yml index 725b3c9..ee27519 100644 --- a/.github/workflows/lint_check.yml +++ b/.github/workflows/lint_check.yml @@ -15,10 +15,10 @@ jobs: with: ref: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} - - name: Set up python 3.7 + - name: Set up python 3.8 uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.8 - name: Run lint check for minimum dependency generator run: | python -m pip install -r test-requirements.txt diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 2dbb1b1..f189c6f 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -15,10 +15,10 @@ jobs: with: ref: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} - - name: Set up python 3.7 + - name: Set up python 3.8 uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.8 - name: Run unit tests for minimum dependency generator run: | python -m pip install -r requirements.txt diff --git a/Makefile b/Makefile index 8eadacd..7cde130 100644 --- a/Makefile +++ b/Makefile @@ -7,12 +7,12 @@ clean: .PHONY: lint lint: - flake8 minimum_dependency_generator/ isort --check-only minimum_dependency_generator/ + black minimum_dependency_generator -t py310 --check .PHONY: lint-fix lint-fix: - autopep8 --in-place --recursive --max-line-length=100 minimum_dependency_generator/ + black minimum_dependency_generator -t py310 isort minimum_dependency_generator/ .PHONY: test @@ -22,3 +22,8 @@ test: .PHONY: testcoverage testcoverage: pytest minimum_dependency_generator/ --cov=minimum_dependency_generator --cov-config=.coveragerc --cache-clear --show-capture=stderr + +.PHONY: installdeps +installdeps: + pip install -r requirements.txt + pip install -r test-requirements.txt diff --git a/minimum_dependency_generator/main.py b/minimum_dependency_generator/main.py index dab4354..5b15733 100644 --- a/minimum_dependency_generator/main.py +++ b/minimum_dependency_generator/main.py @@ -4,22 +4,35 @@ def main(): - parser = ArgumentParser(description="reads a requirements file and outputs the minimized requirements") - - parser.add_argument('--paths', nargs='+', - help='path for requirements to minimize', required=True) - - parser.add_argument('--options', nargs='+', default=None, - help='path for requirements to minimize') - - parser.add_argument('--extras_require', nargs='+', default=None, - help='path for requirements to minimize') - - parser.add_argument('--output_filepath', default=None, - help='path to output minimum dependencies (optional)') + parser = ArgumentParser( + description="reads a requirements file and outputs the minimized requirements" + ) + + parser.add_argument( + "--paths", nargs="+", help="path for requirements to minimize", required=True + ) + + parser.add_argument( + "--options", nargs="+", default=None, help="path for requirements to minimize" + ) + + parser.add_argument( + "--extras_require", + nargs="+", + default=None, + help="path for requirements to minimize", + ) + + parser.add_argument( + "--output_filepath", + default=None, + help="path to output minimum dependencies (optional)", + ) args = parser.parse_args() - requirements = generate_min_requirements(args.paths, args.options, args.extras_require, args.output_filepath) + requirements = generate_min_requirements( + args.paths, args.options, args.extras_require, args.output_filepath + ) requirements = sanitize_string(requirements) # DO NOT remove, the GH action needs to output print("::set-output name=min_reqs::{}".format(requirements)) @@ -29,5 +42,5 @@ def sanitize_string(s): return s.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/minimum_dependency_generator/minimum_dependency_generator.py b/minimum_dependency_generator/minimum_dependency_generator.py index abffbcc..af566b9 100644 --- a/minimum_dependency_generator/minimum_dependency_generator.py +++ b/minimum_dependency_generator/minimum_dependency_generator.py @@ -30,18 +30,19 @@ def remove_comment(requirement): def is_requirement_path(requirement): - if '.txt' in requirement and '-r' in requirement: + if ".txt" in requirement and "-r" in requirement: return True return False def find_operator_version(package, operator): - version = None + matching_versions = [] for x in package.specifier: if x.operator == operator: - version = x.version - break - return version + matching_versions.append(Specifier(operator + x.version)) + # matching_versions is sorted lowest to highest + matching_versions = sorted(matching_versions, key=str) + return matching_versions[-1].version def determine_package_name(package): @@ -51,13 +52,13 @@ def determine_package_name(package): return name -def clean_section(section): - section = section.split('\n') +def clean_cfg_section(section): + section = section.split("\n") section = [x for x in section if len(x) > 1] return section -def find_min_requirement(requirement, python_version="3.7", major_python_version="py3"): +def find_min_requirement(requirement): if is_requirement_path(requirement): # skip requirement paths # ex '-r core_requirements.txt' @@ -67,6 +68,7 @@ def find_min_requirement(requirement, python_version="3.7", major_python_version return if ">=" in requirement: # mininum version specified (ex - 'package >= 0.0.4') + number_operators = requirement.count(">=") package = Requirement(requirement) version = find_operator_version(package, ">=") mininum = create_strict_min(version) @@ -87,9 +89,9 @@ def find_min_requirement(requirement, python_version="3.7", major_python_version def clean_list_length_one(item): - if isinstance(item, list) and len(item) == 1 and ' ' in item[0]: - item = item[0].split(' ') - if item == ['']: + if isinstance(item, list) and len(item) == 1 and " " in item[0]: + item = item[0].split(" ") + if item == [""]: return None return item @@ -113,10 +115,10 @@ def parse_setup_cfg(paths, options, extras_require): extras_require = clean_list_length_one(extras_require) if options and len(options) > 0: for option in options: - requirements += clean_section(config['options'][option]) + requirements += clean_cfg_section(config["options"][option]) if extras_require and len(extras_require) > 0: for extra in extras_require: - requirements += clean_section(config['options.extras_require'][extra]) + requirements += clean_cfg_section(config["options.extras_require"][extra]) return requirements @@ -129,20 +131,30 @@ def parse_pyproject_toml(paths, options, extras_require): extras_require = clean_list_length_one(extras_require) if options and len(options) > 0: for option in options: - requirements += toml_dict['project'][option] + requirements += toml_dict["project"][option] if extras_require and len(extras_require) > 0: for extra in extras_require: - requirements += toml_dict['project']['optional-dependencies'][extra] + requirements += toml_dict["project"]["optional-dependencies"][extra] return requirements -def generate_min_requirements(paths, options=None, extras_require=None, output_filepath=None): +def generate_min_requirements( + paths, options=None, extras_require=None, output_filepath=None +): requirements_to_specifier = defaultdict(list) min_requirements = [] - if len(paths) == 1 and paths[0].endswith('.cfg') and os.path.basename(paths[0]).startswith('setup'): + if ( + len(paths) == 1 + and paths[0].endswith(".cfg") + and os.path.basename(paths[0]).startswith("setup") + ): requirements = parse_setup_cfg(paths, options, extras_require) - elif len(paths) == 1 and paths[0].endswith('.toml') and os.path.basename(paths[0]).startswith('pyproject'): + elif ( + len(paths) == 1 + and paths[0].endswith(".toml") + and os.path.basename(paths[0]).startswith("pyproject") + ): requirements = parse_pyproject_toml(paths, options, extras_require) else: requirements = parse_requirements_text_file(paths) @@ -164,7 +176,7 @@ def generate_min_requirements(paths, options=None, extras_require=None, output_f for req in list(sorted(requirements_to_specifier.values())): min_package = find_min_requirement(req) min_requirements.append(str(min_package)) - min_requirements = '\n'.join(min_requirements) + '\n' + min_requirements = "\n".join(min_requirements) + "\n" if output_filepath: write_file(min_requirements, output_filepath) return min_requirements @@ -172,7 +184,7 @@ def generate_min_requirements(paths, options=None, extras_require=None, output_f def write_file(data, filepath): try: - with open(filepath, 'w') as f: + with open(filepath, "w") as f: f.write(data) except OSError: print("Error writing file") diff --git a/minimum_dependency_generator/tests/conftest.py b/minimum_dependency_generator/tests/conftest.py index a2162f0..4e3bc87 100644 --- a/minimum_dependency_generator/tests/conftest.py +++ b/minimum_dependency_generator/tests/conftest.py @@ -47,8 +47,20 @@ def p_ytest_dep(): @pytest.fixture(scope="session", autouse=True) -def cfg_str(dask_dep, pandas_dep, woodwork_dep, numpy_lower, ploty_dep, numpy_upper, p_ytest_dep): - setup_cfg_str = f'''\ +def scipy_lower(): + return "scipy >= 1.3.3" + + +@pytest.fixture(scope="session", autouse=True) +def scipy_even_higher(): + return "scipy >= 1.5.0" + + +@pytest.fixture(scope="session", autouse=True) +def cfg_str( + dask_dep, pandas_dep, woodwork_dep, numpy_lower, ploty_dep, numpy_upper, p_ytest_dep +): + setup_cfg_str = f"""\ [metadata] name = example_package @@ -65,13 +77,15 @@ def cfg_str(dask_dep, pandas_dep, woodwork_dep, numpy_lower, ploty_dep, numpy_up {numpy_upper} test = {p_ytest_dep} - ''' + """ return setup_cfg_str @pytest.fixture(scope="session", autouse=True) -def toml_cfg(pandas_dep, woodwork_dep, numpy_upper, ploty_dep, p_ytest_dep, dask_dep, numpy_lower): - pyproject_str = f'''\ +def toml_cfg( + pandas_dep, woodwork_dep, numpy_upper, ploty_dep, p_ytest_dep, dask_dep, numpy_lower +): + pyproject_str = f"""\ [project] name = "example_package" requires-python = ">=3.7,<3.10" @@ -90,5 +104,5 @@ def toml_cfg(pandas_dep, woodwork_dep, numpy_upper, ploty_dep, p_ytest_dep, dask "{dask_dep}", "{numpy_lower}", ] - ''' + """ return pyproject_str diff --git a/minimum_dependency_generator/tests/test_cfg_toml.py b/minimum_dependency_generator/tests/test_cfg_toml.py index d7b56cb..f04ae34 100644 --- a/minimum_dependency_generator/tests/test_cfg_toml.py +++ b/minimum_dependency_generator/tests/test_cfg_toml.py @@ -7,13 +7,11 @@ @pytest.mark.parametrize( "file_prefix,file_extension,options", - [('setup', 'cfg', 'install_requires'), ('pyproject', 'toml', 'dependencies')], + [("setup", "cfg", "install_requires"), ("pyproject", "toml", "dependencies")], ) -def test_with_toml_cfg( - file_prefix, file_extension, options, cfg_str, toml_cfg -): +def test_with_toml_cfg(file_prefix, file_extension, options, cfg_str, toml_cfg): file_str = toml_cfg - if file_prefix == 'setup': + if file_prefix == "setup": file_str = cfg_str with tempfile.NamedTemporaryFile( mode="w", suffix="." + file_extension, prefix=file_prefix @@ -23,17 +21,17 @@ def test_with_toml_cfg( paths = [pyproject_file.name] options = [options] - extra_requires = ['test dev'] + extra_requires = ["test dev"] min_requirements = generate_min_requirements(paths, options, extra_requires) verify_min_reqs_cfg_toml(min_requirements) def verify_min_reqs_cfg_toml(min_requirements): - assert '-r' not in min_requirements - assert '.txt' not in min_requirements - assert 'core-requirements.txt' not in min_requirements - min_requirements = min_requirements.split('\n') - assert min_requirements[-1] == '' + assert "-r" not in min_requirements + assert ".txt" not in min_requirements + assert "core-requirements.txt" not in min_requirements + min_requirements = min_requirements.split("\n") + assert min_requirements[-1] == "" min_requirements = min_requirements[:-1] expected_min_reqs = [ "dask[dataframe]==2.30.0", diff --git a/minimum_dependency_generator/tests/test_helpers.py b/minimum_dependency_generator/tests/test_helpers.py index 5f8e8b9..cd07d45 100644 --- a/minimum_dependency_generator/tests/test_helpers.py +++ b/minimum_dependency_generator/tests/test_helpers.py @@ -20,6 +20,13 @@ def test_lower_upper_bound(dask_dep): verify_mininum(mininum_package, "dask", "2.30.0", required_extra="dataframe") +def test_two_lower_bounds(): + mininum_package = find_min_requirement("scipy>=1.3.3,>=1.5.0") + verify_mininum(mininum_package, "scipy", "1.5.0") + mininum_package = find_min_requirement("scipy>=1.5.0,>=1.3.3") + verify_mininum(mininum_package, "scipy", "1.5.0") + + def test_spacing(): mininum_package = find_min_requirement("statsmodels >= 0.12.2") verify_mininum(mininum_package, "statsmodels", "0.12.2") diff --git a/minimum_dependency_generator/tests/test_requirements_text_files.py b/minimum_dependency_generator/tests/test_requirements_text_files.py index bacbd7f..a732e06 100644 --- a/minimum_dependency_generator/tests/test_requirements_text_files.py +++ b/minimum_dependency_generator/tests/test_requirements_text_files.py @@ -4,11 +4,23 @@ def test_with_requirements_texts( - ploty_dep, dask_dep, pandas_dep, woodwork_dep, numpy_upper, numpy_lower, other_req_path + ploty_dep, + dask_dep, + pandas_dep, + woodwork_dep, + numpy_upper, + numpy_lower, + other_req_path, + scipy_lower, + scipy_even_higher, ): min_requirements = [] - requirements_core = "\n".join([dask_dep, pandas_dep, woodwork_dep, numpy_upper]) - requirements_koalas = "\n".join([ploty_dep, numpy_lower, other_req_path]) + requirements_core = "\n".join( + [dask_dep, pandas_dep, woodwork_dep, numpy_upper, scipy_lower] + ) + requirements_koalas = "\n".join( + [ploty_dep, numpy_lower, other_req_path, scipy_even_higher] + ) with tempfile.NamedTemporaryFile( mode="w", suffix=".txt", prefix="out_requirements" ) as _: @@ -23,16 +35,14 @@ def test_with_requirements_texts( koalas_f.writelines(requirements_koalas) koalas_f.flush() paths = [core_f.name, koalas_f.name] - paths = [' '.join(paths)] - min_requirements = generate_min_requirements( - paths=paths - ) + paths = [" ".join(paths)] + min_requirements = generate_min_requirements(paths=paths) assert isinstance(min_requirements, str) - assert '-r' not in min_requirements - assert '.txt' not in min_requirements - assert 'core-requirements.txt' not in min_requirements - min_requirements = min_requirements.split('\n') - assert min_requirements[-1] == '' + assert "-r" not in min_requirements + assert ".txt" not in min_requirements + assert "core-requirements.txt" not in min_requirements + min_requirements = min_requirements.split("\n") + assert min_requirements[-1] == "" min_requirements = min_requirements[:-1] expected_min_reqs = [ "dask[dataframe]==2.30.0", @@ -40,6 +50,7 @@ def test_with_requirements_texts( "pandas==0.24.1", "plotly==4.14.0", "woodwork==0.0.11", + "scipy==1.5.0", ] expected_min_reqs = sorted(expected_min_reqs) assert len(min_requirements) == len(expected_min_reqs) diff --git a/minimum_dependency_generator/tests/test_utils.py b/minimum_dependency_generator/tests/test_utils.py index fd9dc7e..4866df9 100644 --- a/minimum_dependency_generator/tests/test_utils.py +++ b/minimum_dependency_generator/tests/test_utils.py @@ -4,12 +4,12 @@ def test_write_text_file(): - reqs = ['scipy==0.8.0', 'pandas==1.2.0'] - reqs = '\n'.join(reqs) + '\n' + reqs = ["scipy==0.8.0", "pandas==1.2.0"] + reqs = "\n".join(reqs) + "\n" with NamedTemporaryFile() as temp: write_file(reqs, temp.name) with open(temp.name) as written: min_reqs = written.readlines() assert len(min_reqs) == 2 - assert 'scipy==0.8.0' == min_reqs[0].strip() - assert 'pandas==1.2.0' == min_reqs[1].strip() + assert "scipy==0.8.0" == min_reqs[0].strip() + assert "pandas==1.2.0" == min_reqs[1].strip() diff --git a/setup.cfg b/setup.cfg index 47f9730..44d5d90 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,11 +6,6 @@ python_files = minimum_dependency_generator/tests/* filterwarnings = ignore::DeprecationWarning ignore::PendingDeprecationWarning -[flake8] -exclude = docs/* -ignore = E501,W504 -per-file-ignores = - **/__init__.py:F401 [aliases] test=pytest [isort] diff --git a/test-requirements.txt b/test-requirements.txt index 1166fb9..62dd56e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,6 +2,5 @@ pip>=20.2.2 pytest==6.2.4 pytest-cov==2.11.1 isort==5.8.0 -autopep8==1.5.7 -flake8==3.9.1 +black==22.6.0 codecov==2.1.11