diff --git a/CHANGELOG.md b/CHANGELOG.md index d0e1190e69..c19839df49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,11 +21,13 @@ - Added stub test creation to `create_test_yml` ([#2476](https://github.com/nf-core/tools/pull/2476)) - Replace ModulePatch by ComponentPatch ([#2482](https://github.com/nf-core/tools/pull/2482)) - Fixed `nf-core modules lint` to work with new module structure for nf-test ([#2494](https://github.com/nf-core/tools/pull/2494)) +- Add option `--migrate-pytest` to create a module with nf-test taking into account an existing module ([#2549](https://github.com/nf-core/tools/pull/2549)) ### Subworkflows - Added stub test creation to `create_test_yml` ([#2476](https://github.com/nf-core/tools/pull/2476)) - Fixed `nf-core subworkflows lint` to work with new module structure for nf-test ([#2494](https://github.com/nf-core/tools/pull/2494)) +- Add option `--migrate-pytest` to create a subworkflow with nf-test taking into account an existing subworkflow ([#2549](https://github.com/nf-core/tools/pull/2549)) ### General diff --git a/nf_core/__main__.py b/nf_core/__main__.py index b3728999c8..dfdd60f428 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -815,8 +815,20 @@ def modules_remove(ctx, dir, tool): default=False, help="Create a module from the template without TODOs or examples", ) +@click.option("--migrate-pytest", is_flag=True, default=False, help="Migrate a module with pytest tests to nf-test") def create_module( - ctx, tool, dir, author, label, meta, no_meta, force, conda_name, conda_package_version, empty_template + ctx, + tool, + dir, + author, + label, + meta, + no_meta, + force, + conda_name, + conda_package_version, + empty_template, + migrate_pytest, ): """ Create a new DSL2 module from the nf-core template. @@ -841,7 +853,7 @@ def create_module( # Run function try: module_create = ModuleCreate( - dir, tool, author, label, has_meta, force, conda_name, conda_package_version, empty_template + dir, tool, author, label, has_meta, force, conda_name, conda_package_version, empty_template, migrate_pytest ) module_create.create() except UserWarning as e: @@ -1033,7 +1045,8 @@ def bump_versions(ctx, tool, dir, all, show_all): @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @click.option("-a", "--author", type=str, metavar="", help="Module author's GitHub username prefixed with '@'") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist") -def create_subworkflow(ctx, subworkflow, dir, author, force): +@click.option("--migrate-pytest", is_flag=True, default=False, help="Migrate a module with pytest tests to nf-test") +def create_subworkflow(ctx, subworkflow, dir, author, force, migrate_pytest): """ Create a new subworkflow from the nf-core template. @@ -1047,7 +1060,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): # Run function try: - subworkflow_create = SubworkflowCreate(dir, subworkflow, author, force) + subworkflow_create = SubworkflowCreate(dir, subworkflow, author, force, migrate_pytest) subworkflow_create.create() except UserWarning as e: log.critical(e) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 54fade29a2..2105104922 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -9,6 +9,7 @@ import logging import os import re +import shutil import subprocess from pathlib import Path from typing import Dict, Optional @@ -16,11 +17,13 @@ import jinja2 import questionary import rich +import yaml from packaging.version import parse as parse_version import nf_core import nf_core.utils from nf_core.components.components_command import ComponentCommand +from nf_core.lint_utils import run_prettier_on_file log = logging.getLogger(__name__) @@ -38,6 +41,7 @@ def __init__( conda_name: Optional[str] = None, conda_version: Optional[str] = None, empty_template: bool = False, + migrate_pytest: bool = False, ): super().__init__(component_type, directory) self.directory = directory @@ -58,6 +62,7 @@ def __init__( self.docker_container = None self.file_paths: Dict[str, str] = {} self.not_empty_template = not empty_template + self.migrate_pytest = migrate_pytest def create(self): """ @@ -136,20 +141,35 @@ def create(self): # Check existence of directories early for fast-fail self.file_paths = self._get_component_dirs() - if self.component_type == "modules": - # Try to find a bioconda package for 'component' - self._get_bioconda_tool() + if self.migrate_pytest: + # Rename the component directory to old + component_old_dir = self.component_dir + "_old" + component_parent_path = Path(self.directory, self.component_type, self.org) + component_old_path = component_parent_path / component_old_dir + component_path = component_parent_path / self.component_dir - # Prompt for GitHub username - self._get_username() + component_path.rename(component_old_path) + else: + if self.component_type == "modules": + # Try to find a bioconda package for 'component' + self._get_bioconda_tool() - if self.component_type == "modules": - self._get_module_structure_components() + # Prompt for GitHub username + self._get_username() + + if self.component_type == "modules": + self._get_module_structure_components() # Create component template with jinja2 self._render_template() log.info(f"Created component template: '{self.component_name}'") + if self.migrate_pytest: + self._copy_old_files(component_old_path) + log.info("Migrate pytest tests: Copied original module files to new module") + shutil.rmtree(component_old_path) + self._print_and_delete_pytest_files() + new_files = list(self.file_paths.values()) log.info("Created following files:\n " + "\n ".join(new_files)) @@ -348,7 +368,7 @@ def _get_component_dirs(self): component_dir = os.path.join(self.directory, self.component_type, self.org, self.component_dir) # Check if module/subworkflow directories exist already - if os.path.exists(component_dir) and not self.force_overwrite: + if os.path.exists(component_dir) and not self.force_overwrite and not self.migrate_pytest: raise UserWarning( f"{self.component_type[:-1]} directory exists: '{component_dir}'. Use '--force' to overwrite" ) @@ -358,7 +378,7 @@ def _get_component_dirs(self): parent_tool_main_nf = os.path.join( self.directory, self.component_type, self.org, self.component, "main.nf" ) - if self.subtool and os.path.exists(parent_tool_main_nf): + if self.subtool and os.path.exists(parent_tool_main_nf) and not self.migrate_pytest: raise UserWarning( f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.component_name}'" ) @@ -367,7 +387,7 @@ def _get_component_dirs(self): tool_glob = glob.glob( f"{os.path.join(self.directory, self.component_type, self.org, self.component)}/*/main.nf" ) - if not self.subtool and tool_glob: + if not self.subtool and tool_glob and not self.migrate_pytest: raise UserWarning( f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" ) @@ -411,3 +431,61 @@ def _get_username(self): f"[violet]GitHub Username:[/]{' (@author)' if author_default is None else ''}", default=author_default, ) + + def _copy_old_files(self, component_old_path): + """Copy files from old module to new module""" + log.debug("Copying original main.nf file") + shutil.copyfile(component_old_path / "main.nf", self.file_paths[self.component_type + "/main.nf"]) + log.debug("Copying original meta.yml file") + shutil.copyfile(component_old_path / "meta.yml", self.file_paths[self.component_type + "/meta.yml"]) + if self.component_type == "modules": + log.debug("Copying original environment.yml file") + shutil.copyfile( + component_old_path / "environment.yml", self.file_paths[self.component_type + "/environment.yml"] + ) + # Create a nextflow.config file if it contains information other than publishDir + pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir) + nextflow_config = pytest_dir / "nextflow.config" + if nextflow_config.is_file(): + with open(nextflow_config, "r") as fh: + config_lines = "" + for line in fh: + if "publishDir" not in line: + config_lines += line + if len(config_lines) > 0: + log.debug("Copying nextflow.config file from pytest tests") + with open( + Path(self.directory, self.component_type, self.org, self.component_dir, "tests", "nextflow.config"), + "w+", + ) as ofh: + ofh.write(config_lines) + + def _print_and_delete_pytest_files(self): + """Prompt if pytest files should be deleted and printed to stdout""" + pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir) + if rich.prompt.Confirm.ask( + "[violet]Do you want to delete the pytest files?[/]\nPytest file 'main.nf' will be printed to standard output to allow migrating the tests manually to 'main.nf.test'.", + default=False, + ): + with open(pytest_dir / "main.nf", "r") as fh: + log.info(fh.read()) + shutil.rmtree(pytest_dir) + log.info( + "[yellow]Please convert the pytest tests to nf-test in 'main.nf.test'.[/]\n" + "You can find more information about nf-test [link=https://nf-co.re/docs/contributing/modules#migrating-from-pytest-to-nf-test]at the nf-core web[/link]. " + ) + else: + log.info( + "[yellow]Please migrate the pytest tests to nf-test in 'main.nf.test'.[/]\n" + "You can find more information about nf-test [link=https://nf-co.re/docs/contributing/modules#migrating-from-pytest-to-nf-test]at the nf-core web[/link].\n" + f"Once done, make sure to delete the module pytest files to avoid linting errors: {pytest_dir}" + ) + # Delete tags from pytest_modules.yml + modules_yml = Path(self.directory, "tests", "config", "pytest_modules.yml") + with open(modules_yml, "r") as fh: + yml_file = yaml.safe_load(fh) + yml_key = self.component_dir if self.component_type == "modules" else f"subworkflows/{self.component_dir}" + del yml_file[yml_key] + with open(modules_yml, "w") as fh: + yaml.dump(yml_file, fh) + run_prettier_on_file(modules_yml) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index b5368130ce..a5e0795a9f 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -17,6 +17,7 @@ def __init__( conda_name=None, conda_version=None, empty_template=False, + migrate_pytest=False, ): super().__init__( "modules", @@ -29,4 +30,5 @@ def __init__( conda_name, conda_version, empty_template, + migrate_pytest, ) diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index 963076455e..93e9f271be 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -12,6 +12,7 @@ def __init__( component="", author=None, force=False, + migrate_pytest=False, ): super().__init__( "subworkflows", @@ -19,4 +20,5 @@ def __init__( component, author, force=force, + migrate_pytest=migrate_pytest, ) diff --git a/tests/modules/create.py b/tests/modules/create.py index 7cfba484a1..74e5ec3896 100644 --- a/tests/modules/create.py +++ b/tests/modules/create.py @@ -1,11 +1,22 @@ +import filecmp import os +import shutil +from pathlib import Path +from unittest import mock import pytest import requests_cache import responses +import yaml +from git.repo import Repo import nf_core.modules -from tests.utils import mock_anaconda_api_calls, mock_biocontainers_api_calls +from tests.utils import ( + GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH, + GITLAB_URL, + mock_anaconda_api_calls, + mock_biocontainers_api_calls, +) def test_modules_create_succeed(self): @@ -65,3 +76,66 @@ def test_modules_create_nfcore_modules_subtool(self): assert os.path.exists( os.path.join(self.nfcore_modules, "modules", "nf-core", "star", "index", "tests", "main.nf.test") ) + + +@mock.patch("rich.prompt.Confirm.ask") +def test_modules_migrate(self, mock_rich_ask): + """Create a module with the --migrate-pytest option to convert pytest to nf-test""" + pytest_dir = Path(self.nfcore_modules, "tests", "modules", "nf-core", "samtools", "sort") + module_dir = Path(self.nfcore_modules, "modules", "nf-core", "samtools", "sort") + + # Clone modules repo with pytests + shutil.rmtree(self.nfcore_modules) + Repo.clone_from(GITLAB_URL, self.nfcore_modules, branch=GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH) + with open(module_dir / "main.nf", "r") as fh: + old_main_nf = fh.read() + with open(module_dir / "meta.yml", "r") as fh: + old_meta_yml = fh.read() + + # Create a module with --migrate-pytest + mock_rich_ask.return_value = True + module_create = nf_core.modules.ModuleCreate(self.nfcore_modules, "samtools/sort", migrate_pytest=True) + module_create.create() + + with open(module_dir / "main.nf", "r") as fh: + new_main_nf = fh.read() + with open(module_dir / "meta.yml", "r") as fh: + new_meta_yml = fh.read() + nextflow_config = module_dir / "tests" / "nextflow.config" + + # Check that old files have been copied to the new module + assert old_main_nf == new_main_nf + assert old_meta_yml == new_meta_yml + assert nextflow_config.is_file() + + # Check that pytest folder is deleted + assert not pytest_dir.is_dir() + + # Check that pytest_modules.yml is updated + with open(Path(self.nfcore_modules, "tests", "config", "pytest_modules.yml")) as fh: + modules_yml = yaml.safe_load(fh) + assert "samtools/sort" not in modules_yml.keys() + + +@mock.patch("rich.prompt.Confirm.ask") +def test_modules_migrate_no_delete(self, mock_rich_ask): + """Create a module with the --migrate-pytest option to convert pytest to nf-test. + Test that pytest directory is not deleted.""" + pytest_dir = Path(self.nfcore_modules, "tests", "modules", "nf-core", "samtools", "sort") + + # Clone modules repo with pytests + shutil.rmtree(self.nfcore_modules) + Repo.clone_from(GITLAB_URL, self.nfcore_modules, branch=GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH) + + # Create a module with --migrate-pytest + mock_rich_ask.return_value = False + module_create = nf_core.modules.ModuleCreate(self.nfcore_modules, "samtools/sort", migrate_pytest=True) + module_create.create() + + # Check that pytest folder is not deleted + assert pytest_dir.is_dir() + + # Check that pytest_modules.yml is updated + with open(Path(self.nfcore_modules, "tests", "config", "pytest_modules.yml")) as fh: + modules_yml = yaml.safe_load(fh) + assert "samtools/sort" not in modules_yml.keys() diff --git a/tests/subworkflows/create.py b/tests/subworkflows/create.py index 94c2a66331..fc628df34f 100644 --- a/tests/subworkflows/create.py +++ b/tests/subworkflows/create.py @@ -1,8 +1,15 @@ +import filecmp import os +import shutil +from pathlib import Path +from unittest import mock import pytest +import yaml +from git.repo import Repo import nf_core.subworkflows +from tests.utils import GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH, GITLAB_URL def test_subworkflows_create_succeed(self): @@ -35,3 +42,70 @@ def test_subworkflows_create_nfcore_modules(self): assert os.path.exists( os.path.join(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test") ) + + +@mock.patch("rich.prompt.Confirm.ask") +def test_subworkflows_migrate(self, mock_rich_ask): + """Create a subworkflow with the --migrate-pytest option to convert pytest to nf-test""" + pytest_dir = Path(self.nfcore_modules, "tests", "subworkflows", "nf-core", "bam_stats_samtools") + subworkflow_dir = Path(self.nfcore_modules, "subworkflows", "nf-core", "bam_stats_samtools") + + # Clone modules repo with pytests + shutil.rmtree(self.nfcore_modules) + Repo.clone_from(GITLAB_URL, self.nfcore_modules, branch=GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH) + with open(subworkflow_dir / "main.nf", "r") as fh: + old_main_nf = fh.read() + with open(subworkflow_dir / "meta.yml", "r") as fh: + old_meta_yml = fh.read() + + # Create a subworkflow with --migrate-pytest + mock_rich_ask.return_value = True + subworkflow_create = nf_core.subworkflows.SubworkflowCreate( + self.nfcore_modules, "bam_stats_samtools", migrate_pytest=True + ) + subworkflow_create.create() + + with open(subworkflow_dir / "main.nf", "r") as fh: + new_main_nf = fh.read() + with open(subworkflow_dir / "meta.yml", "r") as fh: + new_meta_yml = fh.read() + nextflow_config = subworkflow_dir / "tests" / "nextflow.config" + + # Check that old files have been copied to the new module + assert old_main_nf == new_main_nf + assert old_meta_yml == new_meta_yml + assert nextflow_config.is_file() + + # Check that pytest folder is deleted + assert not pytest_dir.is_dir() + + # Check that pytest_modules.yml is updated + with open(Path(self.nfcore_modules, "tests", "config", "pytest_modules.yml")) as fh: + modules_yml = yaml.safe_load(fh) + assert "subworkflows/bam_stats_samtools" not in modules_yml.keys() + + +@mock.patch("rich.prompt.Confirm.ask") +def test_subworkflows_migrate_no_delete(self, mock_rich_ask): + """Create a subworkflow with the --migrate-pytest option to convert pytest to nf-test. + Test that pytest directory is not deleted.""" + pytest_dir = Path(self.nfcore_modules, "tests", "subworkflows", "nf-core", "bam_stats_samtools") + + # Clone modules repo with pytests + shutil.rmtree(self.nfcore_modules) + Repo.clone_from(GITLAB_URL, self.nfcore_modules, branch=GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH) + + # Create a module with --migrate-pytest + mock_rich_ask.return_value = False + module_create = nf_core.subworkflows.SubworkflowCreate( + self.nfcore_modules, "bam_stats_samtools", migrate_pytest=True + ) + module_create.create() + + # Check that pytest folder is not deleted + assert pytest_dir.is_dir() + + # Check that pytest_modules.yml is updated + with open(Path(self.nfcore_modules, "tests", "config", "pytest_modules.yml")) as fh: + modules_yml = yaml.safe_load(fh) + assert "subworkflows/bam_stats_samtools" not in modules_yml.keys() diff --git a/tests/test_modules.py b/tests/test_modules.py index 21a28d5e46..e2e024a751 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -152,6 +152,8 @@ def test_modulesrepo_class(self): test_modules_create_nfcore_modules, test_modules_create_nfcore_modules_subtool, test_modules_create_succeed, + test_modules_migrate, + test_modules_migrate_no_delete, ) from .modules.info import ( # type: ignore[misc] test_modules_info_in_modules_repo, diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index a4ab959fb4..aaf4080a7b 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -92,6 +92,8 @@ def tearDown(self): test_subworkflows_create_fail_exists, test_subworkflows_create_nfcore_modules, test_subworkflows_create_succeed, + test_subworkflows_migrate, + test_subworkflows_migrate_no_delete, ) from .subworkflows.info import ( # type: ignore[misc] test_subworkflows_info_in_modules_repo,