Merge pull request #1871 from nf-core/dev

Dev > Master for v2.6 release

.github/workflows/rich-codex.yml

@@ -24,8 +24,7 @@ jobs:
         uses: ewels/rich-codex@v1
         env:
           COLUMNS: 100
-          NFCORE_LINT_HIDE_PROGRESS: true
-          NFCORE_MODULES_LINT_HIDE_PROGRESS: true
+          HIDE_PROGRESS: "true"
         with:
           commit_changes: "true"
           clean_img_paths: docs/images/*.svg

docs/api/_src/module_lint_tests/module_changes.md

@@ -0,0 +1,5 @@
+# module_changes
+
+```{eval-rst}
+.. automethod:: nf_core.modules.lint.ModuleLint.module_changes
+```

docs/api/_src/module_lint_tests/module_patch.md

@@ -0,0 +1,5 @@
+# module_patch
+
+```{eval-rst}
+.. automethod:: nf_core.modules.lint.ModuleLint.module_patch
+```

docs/api/_src/module_lint_tests/module_tests.md

@@ -0,0 +1,5 @@
+# module_tests
+
+```{eval-rst}
+.. automethod:: nf_core.modules.lint.ModuleLint.module_tests
+```

docs/api/_src/module_lint_tests/module_version.md

@@ -0,0 +1,5 @@
+# module_version
+
+```{eval-rst}
+.. automethod:: nf_core.modules.lint.ModuleLint.module_version
+```

docs/api/_src/pipeline_lint_tests/modules_structure.md

@@ -0,0 +1,5 @@
+# modules_structure
+
+```{eval-rst}
+.. automethod:: nf_core.lint.PipelineLint.modules_structure
+```

nf_core/lint/__init__.py

@@ -8,7 +8,7 @@
 import datetime
 import json
 import logging
-import re
+import os
 
 import git
 import rich
@@ -172,6 +172,7 @@ class PipelineLint(nf_core.utils.Pipeline):
     from .files_unchanged import files_unchanged
     from .merge_markers import merge_markers
     from .modules_json import modules_json
+    from .modules_structure import modules_structure
     from .multiqc_config import multiqc_config
     from .nextflow_config import nextflow_config
     from .pipeline_name_conventions import pipeline_name_conventions
@@ -227,6 +228,7 @@ def _get_all_lint_tests(release_mode):
             "merge_markers",
             "modules_json",
             "multiqc_config",
+            "modules_structure",
         ] + (["version_consistency"] if release_mode else [])
 
     def _load(self):
@@ -311,7 +313,7 @@ def _lint_pipeline(self):
             rich.progress.BarColumn(bar_width=None),
             "[magenta]{task.completed} of {task.total}[reset] » [bold yellow]{task.fields[test_name]}",
             transient=True,
-            disable=self.hide_progress,
+            disable=self.hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None,
         )
         with self.progress_bar:
             lint_progress = self.progress_bar.add_task(

nf_core/lint/modules_structure.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+import logging
+import os
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+
+def modules_structure(self):
+    """
+    Check that the structure of the modules directory in a pipeline is the correct one:
+        modules/nf-core/TOOL/SUBTOOL
+
+    Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting:
+        modules/nf-core/modules/TOOL/SUBTOOL
+    """
+    wrong_location_modules = []
+    for directory, _, files in os.walk(Path(self.wf_path, "modules")):
+        if "main.nf" in files:
+            module_path = Path(directory).relative_to(Path(self.wf_path, "modules"))
+            parts = module_path.parts
+            # Check that there are modules installed directly under the 'modules' directory
+            if parts[1] == "modules":
+                wrong_location_modules.append(module_path)
+    # If there are modules installed in the wrong location
+    failed = []
+    passed = []
+    if len(wrong_location_modules) > 0:
+        failed = ["modules directory structure is outdated. Should be 'modules/nf-core/TOOL/SUBTOOL'"]
+    else:
+        passed = ["modules directory structure is correct 'modules/nf-core/TOOL/SUBTOOL'"]
+    return {"passed": passed, "warned": [], "failed": failed, "ignored": []}

nf_core/modules/bump_versions.py

@@ -7,8 +7,8 @@
 from __future__ import print_function
 
 import logging
+import os
 import re
-from pathlib import Path
 
 import questionary
 import rich
@@ -55,6 +55,9 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False):
         self.ignored = []
         self.show_up_to_date = show_uptodate
 
+        # Check modules directory structure
+        self.check_modules_structure()
+
         # Verify that this is not a pipeline
         self.dir, repo_type = nf_core.modules.module_utils.get_repo_type(self.dir)
         if not repo_type == "modules":
@@ -101,6 +104,7 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False):
             rich.progress.BarColumn(bar_width=None),
             "[magenta]{task.completed} of {task.total}[reset] » [bold yellow]{task.fields[test_name]}",
             transient=True,
+            disable=os.environ.get("HIDE_PROGRESS", None) is not None,
         )
         with progress_bar:
             bump_progress = progress_bar.add_task(

nf_core/modules/create.py

@@ -88,6 +88,9 @@ def create(self):
         and matching Docker / Singularity images from BioContainers.
         """
 
+        # Check modules directory structure
+        self.check_modules_structure()
+
         # Check whether the given directory is a nf-core pipeline or a clone of nf-core/modules
         try:
             self.directory, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.directory, self.repo_type)

nf_core/modules/info.py

@@ -72,6 +72,9 @@ def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull):
                 pipeline_dir = None
 
         if self.repo_type == "pipeline":
+            # Check modules directory structure
+            self.check_modules_structure()
+            # Check modules.json up to date
             self.modules_json = ModulesJson(self.dir)
             self.modules_json.check_up_to_date()
         else:

nf_core/modules/install.py

@@ -37,6 +37,9 @@ def install(self, module):
         if not self.has_valid_directory():
             return False
 
+        # Check modules directory structure
+        self.check_modules_structure()
+
         # Verify that 'modules.json' is consistent with the installed modules
         modules_json = ModulesJson(self.dir)
         modules_json.check_up_to_date()

nf_core/modules/lint/__init__.py

@@ -266,7 +266,7 @@ def lint_modules(self, modules, local=False, fix_version=False):
             "[magenta]{task.completed} of {task.total}[reset] » [bold yellow]{task.fields[test_name]}",
             transient=True,
             console=console,
-            disable=self.hide_progress,
+            disable=self.hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None,
         )
         with progress_bar:
             lint_progress = progress_bar.add_task(

nf_core/modules/list.py

@@ -22,6 +22,8 @@ def list_modules(self, keywords=None, print_json=False):
         Get available module names from GitHub tree for repo
         and print as list to stdout
         """
+        # Check modules directory structure
+        self.check_modules_structure()
 
         # Initialise rich table
         table = rich.table.Table()

nf_core/modules/module_test.py

@@ -76,6 +76,8 @@ def run(self):
 
     def _check_inputs(self):
         """Do more complex checks about supplied flags."""
+        # Check modules directory structure
+        self.check_modules_structure()
 
         # Retrieving installed modules
         if self.repo_type == "modules":

nf_core/modules/modules_command.py

@@ -147,3 +147,41 @@ def load_lint_config(self):
                 self.lint_config = yaml.safe_load(fh)
         except FileNotFoundError:
             log.debug(f"No lint config file found: {config_fn}")
+
+    def check_modules_structure(self):
+        """
+        Check that the structure of the modules directory in a pipeline is the correct one:
+            modules/nf-core/TOOL/SUBTOOL
+
+        Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting:
+            modules/nf-core/modules/TOOL/SUBTOOL
+        """
+        if self.repo_type == "pipeline":
+            wrong_location_modules = []
+            for directory, _, files in os.walk(Path(self.dir, "modules")):
+                if "main.nf" in files:
+                    module_path = Path(directory).relative_to(Path(self.dir, "modules"))
+                    parts = module_path.parts
+                    # Check that there are modules installed directly under the 'modules' directory
+                    if parts[1] == "modules":
+                        wrong_location_modules.append(module_path)
+            # If there are modules installed in the wrong location
+            if len(wrong_location_modules) > 0:
+                log.info("The modules folder structure is outdated. Reinstalling modules.")
+                # Remove the local copy of the modules repository
+                log.info(f"Removing '{self.modules_repo.local_repo_dir}'")
+                shutil.rmtree(self.modules_repo.local_repo_dir)
+                self.modules_repo.setup_local_repo(
+                    self.modules_repo.remote_url, self.modules_repo.branch, self.hide_progress
+                )
+                # Move wrong modules to the right directory
+                for module in wrong_location_modules:
+                    modules_dir = Path("modules").resolve()
+                    correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:]))
+                    wrong_dir = Path(modules_dir, module)
+                    shutil.move(wrong_dir, correct_dir)
+                    log.info(f"Moved {wrong_dir} to {correct_dir}.")
+                shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules"))
+                # Regenerate modules.json file
+                modules_json = ModulesJson(self.dir)
+                modules_json.check_up_to_date()

nf_core/modules/modules_json.py

@@ -104,7 +104,6 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None):
         """
         if repos is None:
             repos = {}
-
         # Check if there are any nf-core modules installed
         if (modules_dir / nf_core.modules.modules_repo.NF_CORE_MODULES_NAME).exists():
             repos[nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE] = {}
@@ -235,17 +234,22 @@ def determine_module_branches_and_shas(self, install_dir, remote_url, modules):
                     correct_commit_sha = self.find_correct_commit_sha(module, temp_module_dir, modules_repo)
                 else:
                     correct_commit_sha = self.find_correct_commit_sha(module, module_path, modules_repo)
+                    if correct_commit_sha is None:
+                        # Check in the old path
+                        correct_commit_sha = self.find_correct_commit_sha(
+                            module, repo_path / "modules" / module, modules_repo
+                        )
                 if correct_commit_sha is None:
                     log.info(f"Was unable to find matching module files in the {modules_repo.branch} branch.")
-                    choices = [{"name": "No", "value": None}] + [
+                    choices = [{"name": "No", "value": False}] + [
                         {"name": branch, "value": branch} for branch in (available_branches - tried_branches)
                     ]
                     branch = questionary.select(
-                        "Was the modules installed from a different branch in the remote?",
+                        f"Was the module '{module}' installed from a different branch in the remote?\nSelect 'No' for a local module",
                         choices=choices,
                         style=nf_core.utils.nfcore_question_style,
                     ).unsafe_ask()
-                    if branch is None:
+                    if not branch:
                         action = questionary.select(
                             f"Module is untracked '{module}'. Please select what action to take",
                             choices=[
@@ -385,10 +389,11 @@ def has_git_url_and_modules(self):
             elif (
                 not isinstance(repo_url, str)
                 or repo_url == ""
+                or not repo_url.startswith("http")
                 or not isinstance(repo_entry["modules"], dict)
                 or repo_entry["modules"] == {}
             ):
-                log.warning(f"modules.json entry {repo_entry} has non-string or empty entries for git_url or modules")
+                log.debug(f"modules.json entry {repo_entry} has non-string or empty entries for git_url or modules.")
                 return False
         return True
 
@@ -509,11 +514,13 @@ def check_up_to_date(self):
             repos, _ = self.get_pipeline_module_repositories(self.modules_dir, tracked_repos)
 
             modules_with_repos = (
-                (install_dir, str(dir.relative_to(install_dir)))
+                (
+                    nf_core.modules.module_utils.path_from_remote(repo_url),
+                    str(dir.relative_to(nf_core.modules.module_utils.path_from_remote(repo_url))),
+                )
                 for dir in missing_from_modules_json
-                for repo_url, repo_content in repos.items()
-                for install_dir, modules in repo_content["modules"].items()
-                if nf_core.utils.is_relative_to(dir, install_dir)
+                for repo_url in repos
+                if nf_core.utils.is_relative_to(dir, nf_core.modules.module_utils.path_from_remote(repo_url))
             )
 
             repos_with_modules = {}

nf_core/modules/modules_repo.py

@@ -161,7 +161,7 @@ def setup_local_repo(self, remote, branch, hide_progress=True):
                         rich.progress.BarColumn(bar_width=None),
                         "[bold yellow]{task.fields[state]}",
                         transient=True,
-                        disable=hide_progress,
+                        disable=hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None,
                     )
                     with pbar:
                         self.repo = git.Repo.clone_from(
@@ -186,7 +186,7 @@ def setup_local_repo(self, remote, branch, hide_progress=True):
                         rich.progress.BarColumn(bar_width=None),
                         "[bold yellow]{task.fields[state]}",
                         transient=True,
-                        disable=hide_progress,
+                        disable=hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None,
                     )
                     with pbar:
                         self.repo.remotes.origin.fetch(

nf_core/modules/patch.py

@@ -33,6 +33,9 @@ def param_check(self, module):
             raise UserWarning(f"Module '{Path('modules', module_dir, module)}' does not exist in the pipeline")
 
     def patch(self, module=None):
+        # Check modules directory structure
+        self.check_modules_structure()
+
         self.modules_json.check_up_to_date()
         self.param_check(module)
         modules = self.modules_json.get_all_modules()[self.modules_repo.remote_url]

nf_core/modules/remove.py

@@ -21,6 +21,9 @@ def remove(self, module):
             log.error("You cannot remove a module in a clone of nf-core/modules")
             return False
 
+        # Check modules directory structure
+        self.check_modules_structure()
+
         # Check whether pipeline is valid and with a modules.json file
         self.has_valid_directory()
         self.has_modules_file()

nf_core/modules/test_yml_builder.py

@@ -69,6 +69,8 @@ def run(self):
 
     def check_inputs(self):
         """Do more complex checks about supplied flags."""
+        # Check modules directory structure
+        self.check_modules_structure()
 
         # Get the tool name if not specified
         if self.module_name is None:

nf_core/modules/update.py

@@ -82,6 +82,9 @@ def update(self, module=None):
 
         self._parameter_checks()
 
+        # Check modules directory structure
+        self.check_modules_structure()
+
         # Verify that 'modules.json' is consistent with the installed modules
         self.modules_json.check_up_to_date()