From fb1c7e99c589b2c47ccc71e4b0eea43ce6a7791f Mon Sep 17 00:00:00 2001
From: mirpedrol <mirp.julia@gmail.com>
Date: Tue, 4 Oct 2022 15:06:51 +0200
Subject: [PATCH 01/15] create a function to check the modules folder structure

---
 nf_core/modules/modules_command.py | 31 ++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py
index 7e622b18a2..3197045590 100644
--- a/nf_core/modules/modules_command.py
+++ b/nf_core/modules/modules_command.py
@@ -147,3 +147,34 @@ def load_lint_config(self):
                 self.lint_config = yaml.safe_load(fh)
         except FileNotFoundError:
             log.debug(f"No lint config file found: {config_fn}")
+
+    def check_modules_structure(self, hide_progress):
+        """
+        Check that the structure of the modules directory in a pipeline is the correct one:
+            'modules/nf-core/TOOL/SUBTOOL
+        """
+        if self.repo_type == "pipeline":
+            wrong_location_modules = []
+            for directory, _, files in os.walk(Path(self.dir, "modules")):
+                if "main.nf" in files:
+                    module_path = Path(directory).relative_to(Path(self.dir, "modules"))
+                    parts = module_path.parts
+                    # Check that there are modules installed directly under the 'modules' directory
+                    if len(parts) <= 3 and parts[0] != self.modules_repo.repo_path and parts[0] != "local":
+                        wrong_location_modules.append(module_path.parent)
+            # If there are modules installed in the wrong location
+            if len(wrong_location_modules) > 0:
+                # Remove the local copy of the modules repository
+                log.info(f"Removing '{self.modules_repo.local_repo_dir}'")
+                shutil.rmtree(self.modules_repo.local_repo_dir)
+                self.modules_repo.setup_local_repo(
+                    self.modules_repo.remote_url, self.modules_repo.branch, hide_progress
+                )
+                # Move wrong modules to the right directory
+                for module in wrong_location_modules:
+                    correct_dir = Path("modules", self.modules_repo.repo_path, module)
+                    wrong_dir = Path("modules", module)
+                    wrong_dir.rename(correct_dir)
+                    log.info(f"Moved {wrong_dir} to {correct_dir}.")
+                # Regenerate modules.json file
+                ModulesJson(self.dir).check_up_to_date()

From b679e4a27f1a67257a5119c80814bbabacf7fc6f Mon Sep 17 00:00:00 2001
From: mirpedrol <mirp.julia@gmail.com>
Date: Tue, 4 Oct 2022 15:18:55 +0200
Subject: [PATCH 02/15] check modules directory in all modules commands

---
 nf_core/modules/bump_versions.py    | 3 +++
 nf_core/modules/create.py           | 3 +++
 nf_core/modules/info.py             | 3 +++
 nf_core/modules/install.py          | 3 +++
 nf_core/modules/list.py             | 2 ++
 nf_core/modules/module_test.py      | 2 ++
 nf_core/modules/modules_command.py  | 4 ++--
 nf_core/modules/patch.py            | 3 +++
 nf_core/modules/remove.py           | 3 +++
 nf_core/modules/test_yml_builder.py | 2 ++
 nf_core/modules/update.py           | 3 +++
 11 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py
index 3e318db418..6e08f47574 100644
--- a/nf_core/modules/bump_versions.py
+++ b/nf_core/modules/bump_versions.py
@@ -55,6 +55,9 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False):
         self.ignored = []
         self.show_up_to_date = show_uptodate
 
+        # Check modules directory structure
+        self.check_modules_structure()
+
         # Verify that this is not a pipeline
         self.dir, repo_type = nf_core.modules.module_utils.get_repo_type(self.dir)
         if not repo_type == "modules":
diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py
index e50d2b22fe..8a97b34344 100644
--- a/nf_core/modules/create.py
+++ b/nf_core/modules/create.py
@@ -88,6 +88,9 @@ def create(self):
         and matching Docker / Singularity images from BioContainers.
         """
 
+        # Check modules directory structure
+        self.check_modules_structure()
+
         # Check whether the given directory is a nf-core pipeline or a clone of nf-core/modules
         try:
             self.directory, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.directory, self.repo_type)
diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py
index 6e4f0b2d3a..e5b45931c7 100644
--- a/nf_core/modules/info.py
+++ b/nf_core/modules/info.py
@@ -72,6 +72,9 @@ def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull):
                 pipeline_dir = None
 
         if self.repo_type == "pipeline":
+            # Check modules directory structure
+            self.check_modules_structure()
+            # Check modules.json up to date
             self.modules_json = ModulesJson(self.dir)
             self.modules_json.check_up_to_date()
         else:
diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py
index 616963bbe8..b63dab0deb 100644
--- a/nf_core/modules/install.py
+++ b/nf_core/modules/install.py
@@ -37,6 +37,9 @@ def install(self, module):
         if not self.has_valid_directory():
             return False
 
+        # Check modules directory structure
+        self.check_modules_structure()
+
         # Verify that 'modules.json' is consistent with the installed modules
         modules_json = ModulesJson(self.dir)
         modules_json.check_up_to_date()
diff --git a/nf_core/modules/list.py b/nf_core/modules/list.py
index 81e7929058..c1fe890a7e 100644
--- a/nf_core/modules/list.py
+++ b/nf_core/modules/list.py
@@ -22,6 +22,8 @@ def list_modules(self, keywords=None, print_json=False):
         Get available module names from GitHub tree for repo
         and print as list to stdout
         """
+        # Check modules directory structure
+        self.check_modules_structure()
 
         # Initialise rich table
         table = rich.table.Table()
diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py
index 107b0c9545..6448d741b7 100644
--- a/nf_core/modules/module_test.py
+++ b/nf_core/modules/module_test.py
@@ -76,6 +76,8 @@ def run(self):
 
     def _check_inputs(self):
         """Do more complex checks about supplied flags."""
+        # Check modules directory structure
+        self.check_modules_structure()
 
         # Retrieving installed modules
         if self.repo_type == "modules":
diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py
index 3197045590..c2b6a1b118 100644
--- a/nf_core/modules/modules_command.py
+++ b/nf_core/modules/modules_command.py
@@ -148,7 +148,7 @@ def load_lint_config(self):
         except FileNotFoundError:
             log.debug(f"No lint config file found: {config_fn}")
 
-    def check_modules_structure(self, hide_progress):
+    def check_modules_structure(self):
         """
         Check that the structure of the modules directory in a pipeline is the correct one:
             'modules/nf-core/TOOL/SUBTOOL
@@ -168,7 +168,7 @@ def check_modules_structure(self, hide_progress):
                 log.info(f"Removing '{self.modules_repo.local_repo_dir}'")
                 shutil.rmtree(self.modules_repo.local_repo_dir)
                 self.modules_repo.setup_local_repo(
-                    self.modules_repo.remote_url, self.modules_repo.branch, hide_progress
+                    self.modules_repo.remote_url, self.modules_repo.branch, self.hide_progress
                 )
                 # Move wrong modules to the right directory
                 for module in wrong_location_modules:
diff --git a/nf_core/modules/patch.py b/nf_core/modules/patch.py
index 890b0d4c49..ec5e05a277 100644
--- a/nf_core/modules/patch.py
+++ b/nf_core/modules/patch.py
@@ -33,6 +33,9 @@ def param_check(self, module):
             raise UserWarning(f"Module '{Path('modules', module_dir, module)}' does not exist in the pipeline")
 
     def patch(self, module=None):
+        # Check modules directory structure
+        self.check_modules_structure()
+
         self.modules_json.check_up_to_date()
         self.param_check(module)
         modules = self.modules_json.get_all_modules()[self.modules_repo.remote_url]
diff --git a/nf_core/modules/remove.py b/nf_core/modules/remove.py
index 22ce36d169..9f54efca37 100644
--- a/nf_core/modules/remove.py
+++ b/nf_core/modules/remove.py
@@ -21,6 +21,9 @@ def remove(self, module):
             log.error("You cannot remove a module in a clone of nf-core/modules")
             return False
 
+        # Check modules directory structure
+        self.check_modules_structure()
+
         # Check whether pipeline is valid and with a modules.json file
         self.has_valid_directory()
         self.has_modules_file()
diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py
index d8733803d8..0fda6b64ab 100644
--- a/nf_core/modules/test_yml_builder.py
+++ b/nf_core/modules/test_yml_builder.py
@@ -69,6 +69,8 @@ def run(self):
 
     def check_inputs(self):
         """Do more complex checks about supplied flags."""
+        # Check modules directory structure
+        self.check_modules_structure()
 
         # Get the tool name if not specified
         if self.module_name is None:
diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py
index e1dd2cc022..cd59a4109b 100644
--- a/nf_core/modules/update.py
+++ b/nf_core/modules/update.py
@@ -82,6 +82,9 @@ def update(self, module=None):
 
         self._parameter_checks()
 
+        # Check modules directory structure
+        self.check_modules_structure()
+
         # Verify that 'modules.json' is consistent with the installed modules
         self.modules_json.check_up_to_date()
 

From f33edf9d18e658515f1fc6f9665ba9e236989f76 Mon Sep 17 00:00:00 2001
From: mirpedrol <mirp.julia@gmail.com>
Date: Tue, 4 Oct 2022 15:23:18 +0200
Subject: [PATCH 03/15] fix modules dir check

---
 nf_core/modules/modules_command.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py
index c2b6a1b118..e641727151 100644
--- a/nf_core/modules/modules_command.py
+++ b/nf_core/modules/modules_command.py
@@ -160,7 +160,7 @@ def check_modules_structure(self):
                     module_path = Path(directory).relative_to(Path(self.dir, "modules"))
                     parts = module_path.parts
                     # Check that there are modules installed directly under the 'modules' directory
-                    if len(parts) <= 3 and parts[0] != self.modules_repo.repo_path and parts[0] != "local":
+                    if parts[1] == "modules":
                         wrong_location_modules.append(module_path.parent)
             # If there are modules installed in the wrong location
             if len(wrong_location_modules) > 0:
@@ -172,8 +172,9 @@ def check_modules_structure(self):
                 )
                 # Move wrong modules to the right directory
                 for module in wrong_location_modules:
-                    correct_dir = Path("modules", self.modules_repo.repo_path, module)
-                    wrong_dir = Path("modules", module)
+                    modules_dir = Path("modules").resolve()
+                    correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(module.parts[2:]))
+                    wrong_dir = Path(modules_dir, module)
                     wrong_dir.rename(correct_dir)
                     log.info(f"Moved {wrong_dir} to {correct_dir}.")
                 # Regenerate modules.json file

From d2983ddc2d8ec114ede6a7f7142922f8676b179c Mon Sep 17 00:00:00 2001
From: mirpedrol <mirp.julia@gmail.com>
Date: Tue, 4 Oct 2022 15:48:09 +0200
Subject: [PATCH 04/15] fix path to copy to

---
 nf_core/modules/modules_command.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py
index e641727151..578e9a0860 100644
--- a/nf_core/modules/modules_command.py
+++ b/nf_core/modules/modules_command.py
@@ -161,9 +161,10 @@ def check_modules_structure(self):
                     parts = module_path.parts
                     # Check that there are modules installed directly under the 'modules' directory
                     if parts[1] == "modules":
-                        wrong_location_modules.append(module_path.parent)
+                        wrong_location_modules.append(module_path)
             # If there are modules installed in the wrong location
             if len(wrong_location_modules) > 0:
+                log.info("The modules folder structure is outdated. Reinstalling modules.")
                 # Remove the local copy of the modules repository
                 log.info(f"Removing '{self.modules_repo.local_repo_dir}'")
                 shutil.rmtree(self.modules_repo.local_repo_dir)
@@ -173,9 +174,11 @@ def check_modules_structure(self):
                 # Move wrong modules to the right directory
                 for module in wrong_location_modules:
                     modules_dir = Path("modules").resolve()
-                    correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(module.parts[2:]))
+                    correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:]))
                     wrong_dir = Path(modules_dir, module)
-                    wrong_dir.rename(correct_dir)
+                    shutil.move(wrong_dir, correct_dir)
                     log.info(f"Moved {wrong_dir} to {correct_dir}.")
+                shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules"))
                 # Regenerate modules.json file
-                ModulesJson(self.dir).check_up_to_date()
+                modules_json = ModulesJson(self.dir)
+                modules_json.check_up_to_date()

From 7e9ace4fc324c6c0b1e8b354270a9d300b932230 Mon Sep 17 00:00:00 2001
From: mirpedrol <mirp.julia@gmail.com>
Date: Tue, 4 Oct 2022 17:07:55 +0200
Subject: [PATCH 05/15] fix check_up_to_date function

---
 nf_core/__main__.py             |  2 +-
 nf_core/modules/modules_json.py | 11 ++++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/nf_core/__main__.py b/nf_core/__main__.py
index 9df0b2ff97..30aa3f7edd 100755
--- a/nf_core/__main__.py
+++ b/nf_core/__main__.py
@@ -538,7 +538,7 @@ def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff):
             sys.exit(1)
     except (UserWarning, LookupError) as e:
         log.error(e)
-        sys.exit(1)
+        raise
 
 
 # nf-core modules patch
diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py
index 397d6abcf3..32bb9ac6f6 100644
--- a/nf_core/modules/modules_json.py
+++ b/nf_core/modules/modules_json.py
@@ -104,7 +104,6 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None):
         """
         if repos is None:
             repos = {}
-
         # Check if there are any nf-core modules installed
         if (modules_dir / nf_core.modules.modules_repo.NF_CORE_MODULES_NAME).exists():
             repos[nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE] = {}
@@ -509,11 +508,13 @@ def check_up_to_date(self):
             repos, _ = self.get_pipeline_module_repositories(self.modules_dir, tracked_repos)
 
             modules_with_repos = (
-                (install_dir, str(dir.relative_to(install_dir)))
+                (
+                    nf_core.modules.module_utils.path_from_remote(repo_url),
+                    str(dir.relative_to(nf_core.modules.module_utils.path_from_remote(repo_url))),
+                )
                 for dir in missing_from_modules_json
-                for repo_url, repo_content in repos.items()
-                for install_dir, modules in repo_content["modules"].items()
-                if nf_core.utils.is_relative_to(dir, install_dir)
+                for repo_url in repos
+                if nf_core.utils.is_relative_to(dir, nf_core.modules.module_utils.path_from_remote(repo_url))
             )
 
             repos_with_modules = {}

From ead1a9301d038ba33602bbde8e4086ad132f9031 Mon Sep 17 00:00:00 2001
From: mirpedrol <mirp.julia@gmail.com>
Date: Tue, 4 Oct 2022 17:33:15 +0200
Subject: [PATCH 06/15] make branch No to False instead of None

---
 nf_core/modules/modules_json.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py
index 32bb9ac6f6..59297352d3 100644
--- a/nf_core/modules/modules_json.py
+++ b/nf_core/modules/modules_json.py
@@ -236,7 +236,7 @@ def determine_module_branches_and_shas(self, install_dir, remote_url, modules):
                     correct_commit_sha = self.find_correct_commit_sha(module, module_path, modules_repo)
                 if correct_commit_sha is None:
                     log.info(f"Was unable to find matching module files in the {modules_repo.branch} branch.")
-                    choices = [{"name": "No", "value": None}] + [
+                    choices = [{"name": "No", "value": False}] + [
                         {"name": branch, "value": branch} for branch in (available_branches - tried_branches)
                     ]
                     branch = questionary.select(
@@ -244,7 +244,7 @@ def determine_module_branches_and_shas(self, install_dir, remote_url, modules):
                         choices=choices,
                         style=nf_core.utils.nfcore_question_style,
                     ).unsafe_ask()
-                    if branch is None:
+                    if not branch:
                         action = questionary.select(
                             f"Module is untracked '{module}'. Please select what action to take",
                             choices=[
@@ -384,6 +384,7 @@ def has_git_url_and_modules(self):
             elif (
                 not isinstance(repo_url, str)
                 or repo_url == ""
+                or not repo_url.startswith("http")
                 or not isinstance(repo_entry["modules"], dict)
                 or repo_entry["modules"] == {}
             ):

From fb66b45c4d3d5d6456b932d6444a428531226e0b Mon Sep 17 00:00:00 2001
From: mirpedrol <mirp.julia@gmail.com>
Date: Tue, 4 Oct 2022 17:42:29 +0200
Subject: [PATCH 07/15] improve warnings

---
 nf_core/modules/modules_json.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py
index 59297352d3..07b6aef454 100644
--- a/nf_core/modules/modules_json.py
+++ b/nf_core/modules/modules_json.py
@@ -235,12 +235,12 @@ def determine_module_branches_and_shas(self, install_dir, remote_url, modules):
                 else:
                     correct_commit_sha = self.find_correct_commit_sha(module, module_path, modules_repo)
                 if correct_commit_sha is None:
-                    log.info(f"Was unable to find matching module files in the {modules_repo.branch} branch.")
+                    log.info(f"Was unable to find matching module {module} files in the {modules_repo.branch} branch.")
                     choices = [{"name": "No", "value": False}] + [
                         {"name": branch, "value": branch} for branch in (available_branches - tried_branches)
                     ]
                     branch = questionary.select(
-                        "Was the modules installed from a different branch in the remote?",
+                        f"Was the module '{module}' installed from a different branch in the remote?",
                         choices=choices,
                         style=nf_core.utils.nfcore_question_style,
                     ).unsafe_ask()
@@ -388,7 +388,7 @@ def has_git_url_and_modules(self):
                 or not isinstance(repo_entry["modules"], dict)
                 or repo_entry["modules"] == {}
             ):
-                log.warning(f"modules.json entry {repo_entry} has non-string or empty entries for git_url or modules")
+                log.debug(f"modules.json entry {repo_entry} has non-string or empty entries for git_url or modules.")
                 return False
         return True
 

From 3401ea5e5d498d14a9d59e91e6ca36e362088a0d Mon Sep 17 00:00:00 2001
From: mirpedrol <mirp.julia@gmail.com>
Date: Tue, 4 Oct 2022 17:45:06 +0200
Subject: [PATCH 08/15] improve warnings

---
 nf_core/modules/modules_json.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py
index 07b6aef454..644a83cc93 100644
--- a/nf_core/modules/modules_json.py
+++ b/nf_core/modules/modules_json.py
@@ -235,12 +235,12 @@ def determine_module_branches_and_shas(self, install_dir, remote_url, modules):
                 else:
                     correct_commit_sha = self.find_correct_commit_sha(module, module_path, modules_repo)
                 if correct_commit_sha is None:
-                    log.info(f"Was unable to find matching module {module} files in the {modules_repo.branch} branch.")
+                    log.info(f"Was unable to find matching module files in the {modules_repo.branch} branch.")
                     choices = [{"name": "No", "value": False}] + [
                         {"name": branch, "value": branch} for branch in (available_branches - tried_branches)
                     ]
                     branch = questionary.select(
-                        f"Was the module '{module}' installed from a different branch in the remote?",
+                        f"Was the module '{module}' installed from a different branch in the remote?\nSelect 'No' for a local module",
                         choices=choices,
                         style=nf_core.utils.nfcore_question_style,
                     ).unsafe_ask()

From d276aa32356f7c552125f00c4f2c18644e31c653 Mon Sep 17 00:00:00 2001
From: mirpedrol <mirp.julia@gmail.com>
Date: Tue, 4 Oct 2022 17:47:47 +0200
Subject: [PATCH 09/15] remove raise from main

---
 nf_core/__main__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nf_core/__main__.py b/nf_core/__main__.py
index 30aa3f7edd..9df0b2ff97 100755
--- a/nf_core/__main__.py
+++ b/nf_core/__main__.py
@@ -538,7 +538,7 @@ def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff):
             sys.exit(1)
     except (UserWarning, LookupError) as e:
         log.error(e)
-        raise
+        sys.exit(1)
 
 
 # nf-core modules patch

From dd5eb6f270801a1ccae5b7c0ab592c0e4e5ad55f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= <mirp.julia@gmail.com>
Date: Tue, 4 Oct 2022 17:52:26 +0200
Subject: [PATCH 10/15] Update comments from code review

Co-authored-by: Phil Ewels <phil.ewels@scilifelab.se>
---
 nf_core/modules/modules_command.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py
index 578e9a0860..116cd1287e 100644
--- a/nf_core/modules/modules_command.py
+++ b/nf_core/modules/modules_command.py
@@ -151,7 +151,10 @@ def load_lint_config(self):
     def check_modules_structure(self):
         """
         Check that the structure of the modules directory in a pipeline is the correct one:
-            'modules/nf-core/TOOL/SUBTOOL
+            modules/nf-core/TOOL/SUBTOOL
+
+        Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting:
+            modules/nf-core/modules/TOOL/SUBTOOL
         """
         if self.repo_type == "pipeline":
             wrong_location_modules = []

From af1edde769aa56139056c18b57bbe00e97f96179 Mon Sep 17 00:00:00 2001
From: mirpedrol <mirp.julia@gmail.com>
Date: Tue, 4 Oct 2022 18:31:52 +0200
Subject: [PATCH 11/15] add linting of modules directory

---
 nf_core/lint/__init__.py          |  2 ++
 nf_core/lint/modules_structure.py | 33 +++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 nf_core/lint/modules_structure.py

diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py
index 7e61de3ac1..e0622c54f9 100644
--- a/nf_core/lint/__init__.py
+++ b/nf_core/lint/__init__.py
@@ -172,6 +172,7 @@ class PipelineLint(nf_core.utils.Pipeline):
     from .files_unchanged import files_unchanged
     from .merge_markers import merge_markers
     from .modules_json import modules_json
+    from .modules_structure import modules_structure
     from .multiqc_config import multiqc_config
     from .nextflow_config import nextflow_config
     from .pipeline_name_conventions import pipeline_name_conventions
@@ -227,6 +228,7 @@ def _get_all_lint_tests(release_mode):
             "merge_markers",
             "modules_json",
             "multiqc_config",
+            "modules_structure",
         ] + (["version_consistency"] if release_mode else [])
 
     def _load(self):
diff --git a/nf_core/lint/modules_structure.py b/nf_core/lint/modules_structure.py
new file mode 100644
index 0000000000..830e0d1f16
--- /dev/null
+++ b/nf_core/lint/modules_structure.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+import logging
+import os
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+
+def modules_structure(self):
+    """
+    Check that the structure of the modules directory in a pipeline is the correct one:
+        modules/nf-core/TOOL/SUBTOOL
+
+    Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting:
+        modules/nf-core/modules/TOOL/SUBTOOL
+    """
+    wrong_location_modules = []
+    for directory, _, files in os.walk(Path(self.wf_path, "modules")):
+        if "main.nf" in files:
+            module_path = Path(directory).relative_to(Path(self.wf_path, "modules"))
+            parts = module_path.parts
+            # Check that there are modules installed directly under the 'modules' directory
+            if parts[1] == "modules":
+                wrong_location_modules.append(module_path)
+    # If there are modules installed in the wrong location
+    failed = []
+    passed = []
+    if len(wrong_location_modules) > 0:
+        failed = ["modules directory structure is outdated. Should be 'modules/nf-core/TOOL/SUBTOOL'"]
+    else:
+        passed = ["modules directory structure is correct 'modules/nf-core/TOOL/SUBTOOL'"]
+    return {"passed": passed, "warned": [], "failed": failed, "ignored": []}

From 16d39097128cd08d646f7762bf520b2cd313e44e Mon Sep 17 00:00:00 2001
From: Phil Ewels <phil@seqera.io>
Date: Tue, 4 Oct 2022 20:42:02 +0200
Subject: [PATCH 12/15] Create new API docs pages

Ran python make_lint_md.py in api docs dir
---
 docs/api/_src/module_lint_tests/module_changes.md      | 5 +++++
 docs/api/_src/module_lint_tests/module_patch.md        | 5 +++++
 docs/api/_src/module_lint_tests/module_tests.md        | 5 +++++
 docs/api/_src/module_lint_tests/module_version.md      | 5 +++++
 docs/api/_src/pipeline_lint_tests/modules_structure.md | 5 +++++
 5 files changed, 25 insertions(+)
 create mode 100644 docs/api/_src/module_lint_tests/module_changes.md
 create mode 100644 docs/api/_src/module_lint_tests/module_patch.md
 create mode 100644 docs/api/_src/module_lint_tests/module_tests.md
 create mode 100644 docs/api/_src/module_lint_tests/module_version.md
 create mode 100644 docs/api/_src/pipeline_lint_tests/modules_structure.md

diff --git a/docs/api/_src/module_lint_tests/module_changes.md b/docs/api/_src/module_lint_tests/module_changes.md
new file mode 100644
index 0000000000..ce2d428ca9
--- /dev/null
+++ b/docs/api/_src/module_lint_tests/module_changes.md
@@ -0,0 +1,5 @@
+# module_changes
+
+```{eval-rst}
+.. automethod:: nf_core.modules.lint.ModuleLint.module_changes
+```
diff --git a/docs/api/_src/module_lint_tests/module_patch.md b/docs/api/_src/module_lint_tests/module_patch.md
new file mode 100644
index 0000000000..dc98d97a6f
--- /dev/null
+++ b/docs/api/_src/module_lint_tests/module_patch.md
@@ -0,0 +1,5 @@
+# module_patch
+
+```{eval-rst}
+.. automethod:: nf_core.modules.lint.ModuleLint.module_patch
+```
diff --git a/docs/api/_src/module_lint_tests/module_tests.md b/docs/api/_src/module_lint_tests/module_tests.md
new file mode 100644
index 0000000000..d10934c0d6
--- /dev/null
+++ b/docs/api/_src/module_lint_tests/module_tests.md
@@ -0,0 +1,5 @@
+# module_tests
+
+```{eval-rst}
+.. automethod:: nf_core.modules.lint.ModuleLint.module_tests
+```
diff --git a/docs/api/_src/module_lint_tests/module_version.md b/docs/api/_src/module_lint_tests/module_version.md
new file mode 100644
index 0000000000..a088b4cc66
--- /dev/null
+++ b/docs/api/_src/module_lint_tests/module_version.md
@@ -0,0 +1,5 @@
+# module_version
+
+```{eval-rst}
+.. automethod:: nf_core.modules.lint.ModuleLint.module_version
+```
diff --git a/docs/api/_src/pipeline_lint_tests/modules_structure.md b/docs/api/_src/pipeline_lint_tests/modules_structure.md
new file mode 100644
index 0000000000..faa39ca77f
--- /dev/null
+++ b/docs/api/_src/pipeline_lint_tests/modules_structure.md
@@ -0,0 +1,5 @@
+# modules_structure
+
+```{eval-rst}
+.. automethod:: nf_core.lint.PipelineLint.modules_structure
+```

From 7ca39d80195b2d24b347e51c4f60e1444378765d Mon Sep 17 00:00:00 2001
From: Phil Ewels <phil@seqera.io>
Date: Tue, 4 Oct 2022 22:15:21 +0200
Subject: [PATCH 13/15] Add env variable HIDE_PROGRESS to hide progress bars

---
 .github/workflows/rich-codex.yml | 3 +--
 nf_core/lint/__init__.py         | 4 ++--
 nf_core/modules/bump_versions.py | 3 ++-
 nf_core/modules/lint/__init__.py | 2 +-
 nf_core/modules/modules_repo.py  | 4 ++--
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml
index 669c138f46..67ec8fb867 100644
--- a/.github/workflows/rich-codex.yml
+++ b/.github/workflows/rich-codex.yml
@@ -24,8 +24,7 @@ jobs:
         uses: ewels/rich-codex@v1
         env:
           COLUMNS: 100
-          NFCORE_LINT_HIDE_PROGRESS: true
-          NFCORE_MODULES_LINT_HIDE_PROGRESS: true
+          HIDE_PROGRESS: "true"
         with:
           commit_changes: "true"
           clean_img_paths: docs/images/*.svg
diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py
index 7e61de3ac1..1ec3b973b6 100644
--- a/nf_core/lint/__init__.py
+++ b/nf_core/lint/__init__.py
@@ -8,7 +8,7 @@
 import datetime
 import json
 import logging
-import re
+import os
 
 import git
 import rich
@@ -311,7 +311,7 @@ def _lint_pipeline(self):
             rich.progress.BarColumn(bar_width=None),
             "[magenta]{task.completed} of {task.total}[reset] » [bold yellow]{task.fields[test_name]}",
             transient=True,
-            disable=self.hide_progress,
+            disable=self.hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None,
         )
         with self.progress_bar:
             lint_progress = self.progress_bar.add_task(
diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py
index 3e318db418..1c2e0f600a 100644
--- a/nf_core/modules/bump_versions.py
+++ b/nf_core/modules/bump_versions.py
@@ -8,7 +8,7 @@
 
 import logging
 import re
-from pathlib import Path
+import os
 
 import questionary
 import rich
@@ -101,6 +101,7 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False):
             rich.progress.BarColumn(bar_width=None),
             "[magenta]{task.completed} of {task.total}[reset] » [bold yellow]{task.fields[test_name]}",
             transient=True,
+            disable=os.environ.get("HIDE_PROGRESS", None) is not None,
         )
         with progress_bar:
             bump_progress = progress_bar.add_task(
diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py
index 91961fde60..034e7b5f4b 100644
--- a/nf_core/modules/lint/__init__.py
+++ b/nf_core/modules/lint/__init__.py
@@ -266,7 +266,7 @@ def lint_modules(self, modules, local=False, fix_version=False):
             "[magenta]{task.completed} of {task.total}[reset] » [bold yellow]{task.fields[test_name]}",
             transient=True,
             console=console,
-            disable=self.hide_progress,
+            disable=self.hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None,
         )
         with progress_bar:
             lint_progress = progress_bar.add_task(
diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py
index b21dd0b9d0..cde6d26044 100644
--- a/nf_core/modules/modules_repo.py
+++ b/nf_core/modules/modules_repo.py
@@ -161,7 +161,7 @@ def setup_local_repo(self, remote, branch, hide_progress=True):
                         rich.progress.BarColumn(bar_width=None),
                         "[bold yellow]{task.fields[state]}",
                         transient=True,
-                        disable=hide_progress,
+                        disable=hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None,
                     )
                     with pbar:
                         self.repo = git.Repo.clone_from(
@@ -186,7 +186,7 @@ def setup_local_repo(self, remote, branch, hide_progress=True):
                         rich.progress.BarColumn(bar_width=None),
                         "[bold yellow]{task.fields[state]}",
                         transient=True,
-                        disable=hide_progress,
+                        disable=hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None,
                     )
                     with pbar:
                         self.repo.remotes.origin.fetch(

From 4d7c8ad0083a662dfea5ebfcebb594a04d37d65c Mon Sep 17 00:00:00 2001
From: Phil Ewels <phil@seqera.io>
Date: Tue, 4 Oct 2022 22:19:12 +0200
Subject: [PATCH 14/15] Fix isort

---
 nf_core/modules/bump_versions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py
index 1c2e0f600a..cd62cf2b7d 100644
--- a/nf_core/modules/bump_versions.py
+++ b/nf_core/modules/bump_versions.py
@@ -7,8 +7,8 @@
 from __future__ import print_function
 
 import logging
-import re
 import os
+import re
 
 import questionary
 import rich

From e37f6ab0240c391862dac921aa86ffe4a3356f9d Mon Sep 17 00:00:00 2001
From: mirpedrol <mirp.julia@gmail.com>
Date: Tue, 4 Oct 2022 23:08:57 +0200
Subject: [PATCH 15/15] try to find module sha with the old repo path structure

---
 nf_core/modules/modules_json.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py
index 644a83cc93..d0b60adf3a 100644
--- a/nf_core/modules/modules_json.py
+++ b/nf_core/modules/modules_json.py
@@ -234,6 +234,11 @@ def determine_module_branches_and_shas(self, install_dir, remote_url, modules):
                     correct_commit_sha = self.find_correct_commit_sha(module, temp_module_dir, modules_repo)
                 else:
                     correct_commit_sha = self.find_correct_commit_sha(module, module_path, modules_repo)
+                    if correct_commit_sha is None:
+                        # Check in the old path
+                        correct_commit_sha = self.find_correct_commit_sha(
+                            module, repo_path / "modules" / module, modules_repo
+                        )
                 if correct_commit_sha is None:
                     log.info(f"Was unable to find matching module files in the {modules_repo.branch} branch.")
                     choices = [{"name": "No", "value": False}] + [