From f201f069b970fbbdbe22764f7dd416e6ffa7d28c Mon Sep 17 00:00:00 2001 From: Luca Pireddu Date: Mon, 17 Jul 2023 15:20:07 +0200 Subject: [PATCH 01/90] fix: cache repo content read from file Solves issue #298 --- lifemonitor/api/models/repositories/files/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lifemonitor/api/models/repositories/files/base.py b/lifemonitor/api/models/repositories/files/base.py index e6496b2e..44959379 100644 --- a/lifemonitor/api/models/repositories/files/base.py +++ b/lifemonitor/api/models/repositories/files/base.py @@ -100,7 +100,7 @@ def has_path(self, path) -> bool: def get_content(self, binary_mode: bool = False) -> Union[str, bytes, None]: if not self._content and self.dir: with open(f"{self.path}", 'rb' if binary_mode else 'r') as f: - return f.read() + self._content = f.read() return self._content @staticmethod From f8ed7c0dc578489c38a78dde40024eab55addeed Mon Sep 17 00:00:00 2001 From: Luca Pireddu Date: Mon, 17 Jul 2023 15:34:40 +0200 Subject: [PATCH 02/90] fix: Document that roc_link in API can point to a GitHub repository Solves issue #290 --- specs/api.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/api.yaml b/specs/api.yaml index d1f98f76..8a729a94 100644 --- a/specs/api.yaml +++ b/specs/api.yaml @@ -2341,7 +2341,8 @@ components: roc_link: type: string description: | - Link to the workflow RO-Crate + Link to the workflow RO-Crate. Here LM will also accept a link + to a GitHub repository containing the RO-Crate contents. example: http://webserver:5000/download?file=ro-crate-galaxy-sortchangecase.crate.zip authorization: type: string From 363d25d3bbad1e22f96d06451d24ce18e48a1313 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 Jul 2023 15:26:34 +0000 Subject: [PATCH 03/90] build(deps): bump cryptography from 41.0.1 to 41.0.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a98ba389..4431e594 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ boto3~=1.24.80 Bcrypt-Flask==1.0.2 giturlparse~=0.10.0 click-option-group~=0.5.5 -cryptography==41.0.1 +cryptography==41.0.2 dnspython==2.2.1 flask-cors==3.0.10 flask-marshmallow~=0.14.0 From bc16bae5ce67ca7de335117d6b117ea5a8208dcd Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 Jul 2023 08:20:04 +0000 Subject: [PATCH 04/90] build(deps): bump PyYAML from 5.4.1 to 6.0.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4431e594..c7180e47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,7 +39,7 @@ python-dotenv~=0.19.0 python-jenkins==1.7.0 python-redis-lock~=4.0.0 PyGithub==1.55 -PyYAML~=5.4.1 +PyYAML~=6.0.1 pika~=1.2.0 redis~=4.5.5 requests~=2.31.0 From c663d0bcdce1db6bbf090be2cc8cccec00ffc678 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 29 Jun 2023 19:43:45 +0200 Subject: [PATCH 05/90] fix: missing repo class export --- lifemonitor/api/models/repositories/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lifemonitor/api/models/repositories/__init__.py b/lifemonitor/api/models/repositories/__init__.py index 199bc990..083b8e4f 100644 --- a/lifemonitor/api/models/repositories/__init__.py +++ b/lifemonitor/api/models/repositories/__init__.py @@ -30,11 +30,13 @@ RepoCloneContextManager) from .local import (LocalGitWorkflowRepository, LocalWorkflowRepository, Base64WorkflowRepository, ZippedWorkflowRepository) +from .templates import WorkflowRepositoryTemplate __all__ = [ "RepositoryFile", "WorkflowRepositoryConfig", "WorkflowFile", "TemplateRepositoryFile", "WorkflowRepository", "WorkflowRepositoryMetadata", "IssueCheckResult", "LocalWorkflowRepository", "LocalGitWorkflowRepository", "Base64WorkflowRepository", "ZippedWorkflowRepository", - "InstallationGithubWorkflowRepository", "GithubWorkflowRepository", "RepoCloneContextManager" + "InstallationGithubWorkflowRepository", "GithubWorkflowRepository", "RepoCloneContextManager", + "WorkflowRepositoryTemplate" ] From 2b09093d6a477631c1612268068d853a5d84f2bd Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 29 Jun 2023 19:49:23 +0200 Subject: [PATCH 06/90] refactor: remove inheritance from base repo class --- .../api/models/repositories/templates/__init__.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lifemonitor/api/models/repositories/templates/__init__.py b/lifemonitor/api/models/repositories/templates/__init__.py index 1f832913..efc42ede 100644 --- a/lifemonitor/api/models/repositories/templates/__init__.py +++ b/lifemonitor/api/models/repositories/templates/__init__.py @@ -36,21 +36,26 @@ logger = logging.getLogger(__name__) -class WorkflowRepositoryTemplate(WorkflowRepository): +class WorkflowRepositoryTemplate(): # template subclasses templates: List[WorkflowRepositoryTemplate] = None - def __init__(self, name: str, local_path: str = None, - data: dict = None, exclude: List[str] = None) -> None: + def __init__(self, data: Optional[Dict[str, str]] = None, + local_path: Optional[str] = None, init_git: bool = False) -> None: + # if local_path is None then create a temporary directory if not local_path: local_path = tempfile.NamedTemporaryFile(dir='/tmp').name - super().__init__(local_path, exclude=exclude) - self._name = name + # init local path + self._local_path = local_path self._files = None + # init default data self._data = self.get_defaults() + # update data with the provided one if data: self._data.update(data) + # set flag to indicate if the template has been initialised as a git repository + self._init_git = init_git self._dirty = True @classmethod From cbda10151ffe76a7c618f0accec38d480f127ddd Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 29 Jun 2023 19:51:24 +0200 Subject: [PATCH 07/90] fix: adapt initialisation of Nextflow template --- lifemonitor/api/models/repositories/templates/nextflow.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lifemonitor/api/models/repositories/templates/nextflow.py b/lifemonitor/api/models/repositories/templates/nextflow.py index a28a2dc4..ed7745a4 100644 --- a/lifemonitor/api/models/repositories/templates/nextflow.py +++ b/lifemonitor/api/models/repositories/templates/nextflow.py @@ -23,7 +23,7 @@ import logging import os import re -from typing import List +from typing import Dict, List, Optional import nf_core.create from lifemonitor.api.models.repositories.files.base import RepositoryFile @@ -53,8 +53,10 @@ class NextflowRepositoryTemplate(WorkflowRepositoryTemplate): - def __init__(self, name: str, local_path: str = None, data: dict = None, exclude: List[str] = None) -> None: - super().__init__(name, local_path, data, exclude) + def __init__(self, data: Optional[Dict[str, str]] = None, + local_path: Optional[str] = None, + init_git: bool = False) -> None: + super().__init__(data=data, local_path=local_path, init_git=init_git) @property def files(self) -> List[RepositoryFile]: From 04178468f68f57cc356b01a1878010157bd2368e Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 29 Jun 2023 19:52:00 +0200 Subject: [PATCH 08/90] feat: add base Snakemake repo template class --- .../repositories/templates/snakemake.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 lifemonitor/api/models/repositories/templates/snakemake.py diff --git a/lifemonitor/api/models/repositories/templates/snakemake.py b/lifemonitor/api/models/repositories/templates/snakemake.py new file mode 100644 index 00000000..4a4d04ae --- /dev/null +++ b/lifemonitor/api/models/repositories/templates/snakemake.py @@ -0,0 +1,28 @@ +# Copyright (c) 2020-2022 CRS4 +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from typing import Dict, Optional +from . import WorkflowRepositoryTemplate + + +class SnakemakeRepositoryTemplate(WorkflowRepositoryTemplate): + + def __init__(self, data: Optional[Dict[str, str]] = None, local_path: Optional[str] = None, init_git: bool = False) -> None: + super().__init__(data=data, local_path=local_path, init_git=init_git) From 19547c2097e7e666bc9d49fb361c6366db893c32 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 29 Jun 2023 19:57:10 +0200 Subject: [PATCH 09/90] fix: wrong level of debug message --- lifemonitor/api/models/rocrate/generators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lifemonitor/api/models/rocrate/generators.py b/lifemonitor/api/models/rocrate/generators.py index c73970cb..2f83196b 100644 --- a/lifemonitor/api/models/rocrate/generators.py +++ b/lifemonitor/api/models/rocrate/generators.py @@ -59,7 +59,7 @@ def generate_crate(workflow_type: str, "ci_workflow": ci_workflow, "lang_version": lang_version } - logger.warning("Config: %r", cfg) + logger.debug("Config: %r", cfg) crate = make_crate(**cfg) crate.write(local_repo_path) return crate From d5c353dc97bd11a1839bfdc02fac25ec3929cd4a Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 29 Jun 2023 19:58:15 +0200 Subject: [PATCH 10/90] fix: adapt template properties --- .../api/models/repositories/templates/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lifemonitor/api/models/repositories/templates/__init__.py b/lifemonitor/api/models/repositories/templates/__init__.py index efc42ede..303110b8 100644 --- a/lifemonitor/api/models/repositories/templates/__init__.py +++ b/lifemonitor/api/models/repositories/templates/__init__.py @@ -72,13 +72,21 @@ def data(self, data: Dict): self._data.update(data) self._dirty = True + @property + def init_git(self) -> bool: + return self._init_git + + @init_git.setter + def init_git(self, init_git: bool): + self._init_git = init_git + @property def _templates_base_path(self) -> str: return "lifemonitor/templates/repositories" @property def template_path(self) -> str: - return os.path.join(self._templates_base_path, self.name) + return os.path.join(self._templates_base_path, self.type) @property def files(self) -> List[TemplateRepositoryFile]: From eb0089eff35cf58821720d81ee6f56624ca22f6a Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 29 Jun 2023 19:59:30 +0200 Subject: [PATCH 11/90] refactor: update generate and write methods --- .../models/repositories/templates/__init__.py | 52 ++++++++++++++----- .../models/repositories/templates/nextflow.py | 7 ++- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/lifemonitor/api/models/repositories/templates/__init__.py b/lifemonitor/api/models/repositories/templates/__init__.py index 303110b8..3d4e551a 100644 --- a/lifemonitor/api/models/repositories/templates/__init__.py +++ b/lifemonitor/api/models/repositories/templates/__init__.py @@ -128,27 +128,55 @@ def get_file(self, name: str) -> TemplateRepositoryFile: return f return None - def generate(self, target_path: Optional[str] = None) -> LocalWorkflowRepository: + def __init_repo_object__(self, target_path: str) -> LocalWorkflowRepository: + # prepare the metadata to initialise the repository + data = { + 'name': self.data.get('workflow_name', 'MyWorkflow'), + 'license': self.data.get('workflow_license', 'MIT'), + 'owner': self.data.get('workflow_author', 'lm'), + 'exclude': self.data.get('exclude', []), + 'remote_url': self.data.get('repo_url', None), + } + # initialise the repository + repo = LocalWorkflowRepository(target_path, **data) \ + if not self.init_git else LocalGitWorkflowRepository(target_path, **data) + # return the repository object + return repo + + def generate(self, target_path: Optional[str] = None) -> WorkflowRepository: target_path = target_path or self.local_path + # create the target directory if it does not exist + if not os.path.exists(target_path): + os.makedirs(target_path, exist_ok=True) + # initialise the target directory as a git repository + if self._init_git and not WorkflowRepository.is_git_repository(target_path): + pygit2.init_repository(target_path, bare=False) + # initialise the repository object + repo = self.__init_repo_object__(target_path) + # render the template files logger.debug("Rendering template files to %s...", target_path) self.write(target_path) logger.debug("Rendering template files to %s... DONE", target_path) - metadata = self.generate_metadata(target_path) - assert isinstance(metadata, WorkflowRepositoryMetadata), "Error generating workflow repository metadata" - return metadata.repository - - def generate_metadata(self, target_path: Optional[str] = None) -> WorkflowRepositoryMetadata: - target_path = target_path or self.local_path - repo = LocalWorkflowRepository(target_path) + # generate the metadata opts = self.data.copy() opts.update({ 'root': target_path, }) - self._metadata = repo.generate_metadata(**opts) - return self._metadata + metadata = repo.generate_metadata(**opts) + assert isinstance(metadata, WorkflowRepositoryMetadata), "Error generating workflow repository metadata" + # return the repository + return repo - def write(self, target_path: str, overwrite: bool = False): - super().write(target_path, overwrite=overwrite) + def write(self, target_path: str, overwrite: bool = False) -> None: + for f in self.files: + base_path = os.path.join(target_path, f.dir) + file_path = os.path.join(base_path, f.name) + os.makedirs(base_path, exist_ok=True) + file_exists = os.path.isfile(file_path) + if not file_exists or overwrite: + logger.debug("%s file: %r", "Overwriting" if file_exists else "Writing", file_path) + with open(file_path, "w") as out: + out.write(f.get_content()) @classmethod def _types(cls) -> List[WorkflowRepositoryTemplate]: diff --git a/lifemonitor/api/models/repositories/templates/nextflow.py b/lifemonitor/api/models/repositories/templates/nextflow.py index ed7745a4..7b9d0e8b 100644 --- a/lifemonitor/api/models/repositories/templates/nextflow.py +++ b/lifemonitor/api/models/repositories/templates/nextflow.py @@ -92,14 +92,17 @@ def generate(self, target_path: str = None) -> LocalWorkflowRepository: file.write(ignore_license) # patch permission of checker script os.chmod(os.path.join(target_path, 'bin/check_samplesheet.py'), 0o777) - logger.debug("Rendering template files to %s... DONE", target_path) - repo = LocalWorkflowRepository(local_path=target_path) + + # create the repository object + repo = self.__init_repo_object__(target_path) + # generate metadata opts = self.data.copy() opts.update({ 'root': target_path, }) repo.generate_metadata(**self.data) + # return the repository object return repo From 58bd1d059e5b94a72c8557b96309ec2728f78723 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 29 Jun 2023 20:00:20 +0200 Subject: [PATCH 12/90] fix: initialisation of template on wizard --- lifemonitor/api/models/wizards/repository_template.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lifemonitor/api/models/wizards/repository_template.py b/lifemonitor/api/models/wizards/repository_template.py index 255bfcd7..277f8687 100644 --- a/lifemonitor/api/models/wizards/repository_template.py +++ b/lifemonitor/api/models/wizards/repository_template.py @@ -58,11 +58,11 @@ def get_files(wizard: RepositoryTemplateWizard, repo: GithubWorkflowRepository, except Exception: workflow_path = target_path - repo_template = WorkflowRepositoryTemplate.new_instance(workflow_type, local_path=workflow_path, data={ + repo_template = WorkflowRepositoryTemplate.new_instance(workflow_type, data={ 'workflow_name': workflow_name, 'workflow_description': workflow_description, 'workflow_version': repo.default_branch, 'repo_url': repo.html_url, 'repo_full_name': repo.full_name, 'main_branch': repo.default_branch - }).generate() + }, local_path=workflow_path).generate() logger.debug("Template files: %r --> %r", repo_template, repo_template.files) logger.debug("Repository files: %r --> %r", repo, repo.files) From ce62c673c8c33880dcb1dc695659f3d6fd093694 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 29 Jun 2023 20:01:17 +0200 Subject: [PATCH 13/90] fix: adapt template properties --- .../api/models/repositories/templates/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lifemonitor/api/models/repositories/templates/__init__.py b/lifemonitor/api/models/repositories/templates/__init__.py index 3d4e551a..36493535 100644 --- a/lifemonitor/api/models/repositories/templates/__init__.py +++ b/lifemonitor/api/models/repositories/templates/__init__.py @@ -62,6 +62,15 @@ def __init__(self, data: Optional[Dict[str, str]] = None, def get_defaults(cls) -> Dict: return {} + @property + def type(self) -> str: + wf_type = self.__class__.__name__.replace("RepositoryTemplate", "").lower() + return wf_type if wf_type != 'workflow' else 'other' + + @property + def local_path(self) -> str: + return self._local_path + @property def data(self) -> Dict: return self._data From ee304d64de39fb00b9501640e10e67671b6fb415 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 29 Jun 2023 20:01:31 +0200 Subject: [PATCH 14/90] fix: missing imports --- lifemonitor/api/models/repositories/templates/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lifemonitor/api/models/repositories/templates/__init__.py b/lifemonitor/api/models/repositories/templates/__init__.py index 36493535..0d62a625 100644 --- a/lifemonitor/api/models/repositories/templates/__init__.py +++ b/lifemonitor/api/models/repositories/templates/__init__.py @@ -25,9 +25,12 @@ import tempfile from typing import Dict, List, Optional +import pygit2 + from lifemonitor.api.models.repositories.files import (TemplateRepositoryFile, WorkflowFile) -from lifemonitor.api.models.repositories.local import LocalWorkflowRepository +from lifemonitor.api.models.repositories.local import ( + LocalGitWorkflowRepository, LocalWorkflowRepository) from lifemonitor.utils import find_types, to_kebab_case from ..base import WorkflowRepository, WorkflowRepositoryMetadata From 4117d87052df50c122ffcc6d1f6115c5436c1da5 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 29 Jun 2023 20:01:59 +0200 Subject: [PATCH 15/90] refactor: update template factory --- lifemonitor/api/models/repositories/templates/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lifemonitor/api/models/repositories/templates/__init__.py b/lifemonitor/api/models/repositories/templates/__init__.py index 0d62a625..9c12d5a4 100644 --- a/lifemonitor/api/models/repositories/templates/__init__.py +++ b/lifemonitor/api/models/repositories/templates/__init__.py @@ -204,8 +204,8 @@ def _get_type(cls, name: str) -> WorkflowRepositoryTemplate: return WorkflowRepositoryTemplate @classmethod - def new_instance(cls, name: str, local_path: str = None, - data: dict = None, exclude: List[str] = None) -> WorkflowRepositoryTemplate: - tmpl_type = cls._get_type(name) + def new_instance(cls, type: str, data: Dict[str, str], + local_path: Optional[str] = None, init_git: bool = False) -> WorkflowRepositoryTemplate: + tmpl_type = cls._get_type(type) logger.debug("Using template type: %s", tmpl_type) - return tmpl_type(name, local_path=local_path, data=data, exclude=exclude) + return tmpl_type(data=data, local_path=local_path, init_git=init_git) From 1ca63eac705123da03a50c59d7f0fb22d2874c9a Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 29 Jun 2023 20:03:18 +0200 Subject: [PATCH 16/90] feat: add base files for generic repo types --- .../repositories/files/workflows/other.py | 35 +++++++++++++++++++ .../templates/repositories/other/workflow | 1 + 2 files changed, 36 insertions(+) create mode 100644 lifemonitor/api/models/repositories/files/workflows/other.py create mode 100644 lifemonitor/templates/repositories/other/workflow diff --git a/lifemonitor/api/models/repositories/files/workflows/other.py b/lifemonitor/api/models/repositories/files/workflows/other.py new file mode 100644 index 00000000..9a4139dc --- /dev/null +++ b/lifemonitor/api/models/repositories/files/workflows/other.py @@ -0,0 +1,35 @@ +# Copyright (c) 2020-2022 CRS4 +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import annotations + +import logging + +from lifemonitor.api.models.repositories.files.workflows import WorkflowFile + +# set module level logger +logger = logging.getLogger(__name__) + + +class OtherWorkflowFile(WorkflowFile): + + FILE_PATTERNS = ( + ("workflow", "", ""), + ) diff --git a/lifemonitor/templates/repositories/other/workflow b/lifemonitor/templates/repositories/other/workflow new file mode 100644 index 00000000..b31ead00 --- /dev/null +++ b/lifemonitor/templates/repositories/other/workflow @@ -0,0 +1 @@ +# empty workflow \ No newline at end of file From c6c4689dfa30c7663cea65dfeba654571a0edc8d Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 29 Jun 2023 20:03:34 +0200 Subject: [PATCH 17/90] test: add unit tests --- .../repositories/test_template_repos.py | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 tests/unit/api/models/repositories/test_template_repos.py diff --git a/tests/unit/api/models/repositories/test_template_repos.py b/tests/unit/api/models/repositories/test_template_repos.py new file mode 100644 index 00000000..6f00f768 --- /dev/null +++ b/tests/unit/api/models/repositories/test_template_repos.py @@ -0,0 +1,95 @@ +# Copyright (c) 2020-2022 CRS4 +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import logging +from typing import Dict, List + +import pytest +import tempfile + +import lifemonitor.api.models.repositories as repos +from lifemonitor.api.models.repositories.templates import WorkflowRepositoryTemplate +from lifemonitor.api.models.repositories.templates.galaxy import GalaxyRepositoryTemplate +from lifemonitor.api.models.repositories.templates.nextflow import NextflowRepositoryTemplate +from lifemonitor.api.models.repositories.templates.snakemake import SnakemakeRepositoryTemplate + +logger = logging.getLogger(__name__) + + +@pytest.fixture +def test_repo_info(simple_local_wf_repo) -> Dict[str, str]: + return { + "name": "MyWorkflowTest", + "owner": "lm", + "description": "My workflow test description", + "license": "MIT", + "exclude": [".*"], + "default_branch": "main", + "active_branch": "main", + "full_name": "lm/MyWorkflowTest", + "remote_url": 'https://github.com/lm/MyWorkflowTest', + } + + +def repo_template_types() -> List[str]: + return ['galaxy', 'snakemake', 'nextflow', 'other'] + + +@pytest.fixture(params=repo_template_types()) +def repo_template_type(request): + return request.param + + +def test_repo_template(test_repo_info, repo_template_type): + + # with tempfile.TemporaryDirectory() as workflow_path: + workflow_path = tempfile.TemporaryDirectory().name + logger.debug("Creating a new Galaxy workflow repository template in %r", workflow_path) + # instantiate the template + tmpl = WorkflowRepositoryTemplate.new_instance(repo_template_type, data={ + 'workflow_name': test_repo_info['name'], 'workflow_description': test_repo_info['description'], + 'workflow_version': '1.0.0', 'workflow_author': 'lm', 'workflow_license': test_repo_info['license'], + 'repo_url': test_repo_info['remote_url'], 'repo_full_name': test_repo_info['full_name'], + 'main_branch': test_repo_info['default_branch'] + }, local_path=workflow_path) + # check the template type + assert isinstance(tmpl, WorkflowRepositoryTemplate), "Template is not a WorkflowRepositoryTemplate" + assert tmpl.type == repo_template_type, "Template type is not correct" + + # check if the template is the expected one + if repo_template_type == 'galaxy': + assert isinstance(tmpl, GalaxyRepositoryTemplate), "Template is not a GalaxyWorkflowTemplate" + if repo_template_type == 'nextflow': + assert isinstance(tmpl, NextflowRepositoryTemplate), "Template is not a SnakeMakeWorkflowTemplate" + if repo_template_type == 'snakemake': + assert isinstance(tmpl, SnakemakeRepositoryTemplate), "Template is not a NextflowWorkflowTemplate" + + # generate the repository + repo = tmpl.generate() + + # check the repository metadata + assert repo, "Repository object is None" + assert isinstance(repo, repos.LocalWorkflowRepository), "Repository is not a WorkflowRepository" + assert repo.name == test_repo_info['name'], "Repository name is not correct" + assert repo.owner == test_repo_info['owner'], "Repository owner is not correct" + assert repo.full_name == f"{test_repo_info['owner']}/{test_repo_info['name']}", "Repository full name is not correct" + assert repo.license == test_repo_info['license'], "Repository license is not correct" + assert repo.local_path == workflow_path, "Repository local path is not correct" + assert repo.remote_url == test_repo_info['remote_url'], "Repository remote url is not correct" From 0857a69f4f813c4f94bd4dfeacb26ac70b053909 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 3 Jul 2023 10:50:42 +0000 Subject: [PATCH 18/90] fix: update username getter --- docker/lifemonitor.Dockerfile | 16 +++++++++------- lifemonitor/utils.py | 17 ++++++++++------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/docker/lifemonitor.Dockerfile b/docker/lifemonitor.Dockerfile index 3083b155..3f8d67f3 100644 --- a/docker/lifemonitor.Dockerfile +++ b/docker/lifemonitor.Dockerfile @@ -10,6 +10,8 @@ RUN apt-get update -q \ # Create a user 'lm' with HOME at /lm and set 'lm' as default git user RUN useradd -d /lm -m lm +# Set the default user +ENV USER=lm # Copy requirements and certificates COPY --chown=lm:lm requirements.txt certs/*.crt /lm/ @@ -45,13 +47,13 @@ COPY \ # Update permissions and install optional certificates RUN chmod 755 \ - /usr/local/bin/wait-for-postgres.sh \ - /usr/local/bin/wait-for-redis.sh \ - /usr/local/bin/wait-for-file.sh \ - /usr/local/bin/lm_entrypoint.sh \ - /usr/local/bin/worker_entrypoint.sh \ - /usr/local/bin/wss-entrypoint.sh \ - /nextflow \ + /usr/local/bin/wait-for-postgres.sh \ + /usr/local/bin/wait-for-redis.sh \ + /usr/local/bin/wait-for-file.sh \ + /usr/local/bin/lm_entrypoint.sh \ + /usr/local/bin/worker_entrypoint.sh \ + /usr/local/bin/wss-entrypoint.sh \ + /nextflow \ && certs=$(ls *.crt 2> /dev/null) \ && mv *.crt /usr/local/share/ca-certificates/ \ && update-ca-certificates || true \ diff --git a/lifemonitor/utils.py b/lifemonitor/utils.py index 67e3df91..ff038a31 100644 --- a/lifemonitor/utils.py +++ b/lifemonitor/utils.py @@ -437,13 +437,16 @@ def isoformat_to_datetime(iso: str) -> datetime: def get_current_username() -> str: try: - import pwd - return pwd.getpwuid(os.getuid()).pw_name - except Exception as e: - logger.warning("Unable to get current username: %s", e) - if logger.isEnabledFor(logging.DEBUG): - logger.exception(e) - return "unknown" + return os.environ["USER"] + except KeyError: + try: + import pwd + return pwd.getpwuid(os.getuid()).pw_name + except Exception as e: + logger.warning("Unable to get current username: %s", e) + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) + return "unknown" def parse_date_interval(interval: str) -> Tuple[Literal['<=', '>=', '<', '>', '..'], Optional[datetime], datetime]: From 761abb7ee375c4fe66ecccf319fed4ecde0441c3 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 3 Jul 2023 10:51:24 +0000 Subject: [PATCH 19/90] refactor: reorder imports --- .../models/repositories/test_template_repos.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/unit/api/models/repositories/test_template_repos.py b/tests/unit/api/models/repositories/test_template_repos.py index 6f00f768..bd194fdc 100644 --- a/tests/unit/api/models/repositories/test_template_repos.py +++ b/tests/unit/api/models/repositories/test_template_repos.py @@ -19,16 +19,13 @@ # SOFTWARE. import logging +import tempfile from typing import Dict, List import pytest -import tempfile import lifemonitor.api.models.repositories as repos -from lifemonitor.api.models.repositories.templates import WorkflowRepositoryTemplate -from lifemonitor.api.models.repositories.templates.galaxy import GalaxyRepositoryTemplate -from lifemonitor.api.models.repositories.templates.nextflow import NextflowRepositoryTemplate -from lifemonitor.api.models.repositories.templates.snakemake import SnakemakeRepositoryTemplate +import lifemonitor.api.models.repositories.templates as templates logger = logging.getLogger(__name__) @@ -63,23 +60,23 @@ def test_repo_template(test_repo_info, repo_template_type): workflow_path = tempfile.TemporaryDirectory().name logger.debug("Creating a new Galaxy workflow repository template in %r", workflow_path) # instantiate the template - tmpl = WorkflowRepositoryTemplate.new_instance(repo_template_type, data={ + tmpl = templates.WorkflowRepositoryTemplate.new_instance(repo_template_type, data={ 'workflow_name': test_repo_info['name'], 'workflow_description': test_repo_info['description'], 'workflow_version': '1.0.0', 'workflow_author': 'lm', 'workflow_license': test_repo_info['license'], 'repo_url': test_repo_info['remote_url'], 'repo_full_name': test_repo_info['full_name'], 'main_branch': test_repo_info['default_branch'] }, local_path=workflow_path) # check the template type - assert isinstance(tmpl, WorkflowRepositoryTemplate), "Template is not a WorkflowRepositoryTemplate" + assert isinstance(tmpl, templates.WorkflowRepositoryTemplate), "Template is not a WorkflowRepositoryTemplate" assert tmpl.type == repo_template_type, "Template type is not correct" # check if the template is the expected one if repo_template_type == 'galaxy': - assert isinstance(tmpl, GalaxyRepositoryTemplate), "Template is not a GalaxyWorkflowTemplate" + assert isinstance(tmpl, templates.galaxy.GalaxyRepositoryTemplate), "Template is not a GalaxyWorkflowTemplate" if repo_template_type == 'nextflow': - assert isinstance(tmpl, NextflowRepositoryTemplate), "Template is not a SnakeMakeWorkflowTemplate" + assert isinstance(tmpl, templates.nextflow.NextflowRepositoryTemplate), "Template is not a SnakeMakeWorkflowTemplate" if repo_template_type == 'snakemake': - assert isinstance(tmpl, SnakemakeRepositoryTemplate), "Template is not a NextflowWorkflowTemplate" + assert isinstance(tmpl, templates.snakemake.SnakemakeRepositoryTemplate), "Template is not a NextflowWorkflowTemplate" # generate the repository repo = tmpl.generate() From 38e7c341f25d06d5720b1b0cef0e96b3fa6f0daa Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 3 Jul 2023 11:07:18 +0000 Subject: [PATCH 20/90] style: fix flake8 issues --- lifemonitor/api/models/repositories/templates/nextflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lifemonitor/api/models/repositories/templates/nextflow.py b/lifemonitor/api/models/repositories/templates/nextflow.py index 7b9d0e8b..9574548a 100644 --- a/lifemonitor/api/models/repositories/templates/nextflow.py +++ b/lifemonitor/api/models/repositories/templates/nextflow.py @@ -93,7 +93,7 @@ def generate(self, target_path: str = None) -> LocalWorkflowRepository: # patch permission of checker script os.chmod(os.path.join(target_path, 'bin/check_samplesheet.py'), 0o777) logger.debug("Rendering template files to %s... DONE", target_path) - + # create the repository object repo = self.__init_repo_object__(target_path) # generate metadata From 19a41c14349f3f80caad0302c9f5ae9951ad2f3d Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 3 Jul 2023 13:39:34 +0000 Subject: [PATCH 21/90] test: fix missing class --- tests/unit/api/models/test_workflow_types.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/api/models/test_workflow_types.py b/tests/unit/api/models/test_workflow_types.py index 08ed8fa3..1d694230 100644 --- a/tests/unit/api/models/test_workflow_types.py +++ b/tests/unit/api/models/test_workflow_types.py @@ -21,6 +21,7 @@ import logging import pytest + from lifemonitor.api.models.repositories.files.base import RepositoryFile from lifemonitor.api.models.repositories.files.workflows import WorkflowFile from lifemonitor.api.models.repositories.files.workflows.galaxy import \ @@ -29,6 +30,8 @@ JupyterWorkflowFile from lifemonitor.api.models.repositories.files.workflows.nextflow import \ NextflowWorkflowFile +from lifemonitor.api.models.repositories.files.workflows.other import \ + OtherWorkflowFile from lifemonitor.api.models.repositories.files.workflows.snakemake import \ SnakemakeWorkflowFile from lifemonitor.api.models.repositories.github import GithubWorkflowRepository @@ -59,7 +62,7 @@ def remote_snakemake_github_repository(github_token) -> GithubWorkflowRepository def test_workflow_types(): - expected_types = (GalaxyWorkflowFile, SnakemakeWorkflowFile, JupyterWorkflowFile, NextflowWorkflowFile) + expected_types = (GalaxyWorkflowFile, SnakemakeWorkflowFile, JupyterWorkflowFile, NextflowWorkflowFile, OtherWorkflowFile) actual_types = WorkflowFile.get_types() assert len(actual_types) == len(expected_types), "Unexpected number of supported workflow types" for w_type in expected_types: From c65b3b441805ac2b411494804b9915e197567998 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 3 Jul 2023 14:53:17 +0000 Subject: [PATCH 22/90] build(pip): pin SQLAlchemy to 1.4.41 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c7180e47..379e600f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -47,7 +47,7 @@ snakemake==6.13.1 networkx~=3.1.0 rich~=13.4.1 rocrate==0.8.0 -SQLAlchemy~=1.4.41 +SQLAlchemy==1.4.41 wheel~=0.40.0 Werkzeug~=2.2.3 repo2rocrate~=0.1.2 From d67449c73958922303566774ca581125c8ebed17 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 3 Jul 2023 16:17:43 +0000 Subject: [PATCH 23/90] test: fix fixture name --- .../repositories/test_template_repos.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/unit/api/models/repositories/test_template_repos.py b/tests/unit/api/models/repositories/test_template_repos.py index bd194fdc..ffc6e343 100644 --- a/tests/unit/api/models/repositories/test_template_repos.py +++ b/tests/unit/api/models/repositories/test_template_repos.py @@ -31,7 +31,7 @@ @pytest.fixture -def test_repo_info(simple_local_wf_repo) -> Dict[str, str]: +def repo_info(simple_local_wf_repo) -> Dict[str, str]: return { "name": "MyWorkflowTest", "owner": "lm", @@ -54,17 +54,17 @@ def repo_template_type(request): return request.param -def test_repo_template(test_repo_info, repo_template_type): +def test_repo_template(repo_info, repo_template_type): # with tempfile.TemporaryDirectory() as workflow_path: workflow_path = tempfile.TemporaryDirectory().name logger.debug("Creating a new Galaxy workflow repository template in %r", workflow_path) # instantiate the template tmpl = templates.WorkflowRepositoryTemplate.new_instance(repo_template_type, data={ - 'workflow_name': test_repo_info['name'], 'workflow_description': test_repo_info['description'], - 'workflow_version': '1.0.0', 'workflow_author': 'lm', 'workflow_license': test_repo_info['license'], - 'repo_url': test_repo_info['remote_url'], 'repo_full_name': test_repo_info['full_name'], - 'main_branch': test_repo_info['default_branch'] + 'workflow_name': repo_info['name'], 'workflow_description': repo_info['description'], + 'workflow_version': '1.0.0', 'workflow_author': 'lm', 'workflow_license': repo_info['license'], + 'repo_url': repo_info['remote_url'], 'repo_full_name': repo_info['full_name'], + 'main_branch': repo_info['default_branch'] }, local_path=workflow_path) # check the template type assert isinstance(tmpl, templates.WorkflowRepositoryTemplate), "Template is not a WorkflowRepositoryTemplate" @@ -84,9 +84,9 @@ def test_repo_template(test_repo_info, repo_template_type): # check the repository metadata assert repo, "Repository object is None" assert isinstance(repo, repos.LocalWorkflowRepository), "Repository is not a WorkflowRepository" - assert repo.name == test_repo_info['name'], "Repository name is not correct" - assert repo.owner == test_repo_info['owner'], "Repository owner is not correct" - assert repo.full_name == f"{test_repo_info['owner']}/{test_repo_info['name']}", "Repository full name is not correct" - assert repo.license == test_repo_info['license'], "Repository license is not correct" + assert repo.name == repo_info['name'], "Repository name is not correct" + assert repo.owner == repo_info['owner'], "Repository owner is not correct" + assert repo.full_name == f"{repo_info['owner']}/{repo_info['name']}", "Repository full name is not correct" + assert repo.license == repo_info['license'], "Repository license is not correct" assert repo.local_path == workflow_path, "Repository local path is not correct" - assert repo.remote_url == test_repo_info['remote_url'], "Repository remote url is not correct" + assert repo.remote_url == repo_info['remote_url'], "Repository remote url is not correct" From 2e9cf950f8b57fe2fea03d85043ec860d1438568 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 3 Jul 2023 16:54:52 +0000 Subject: [PATCH 24/90] remove useless fixture --- tests/unit/api/models/repositories/test_template_repos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/api/models/repositories/test_template_repos.py b/tests/unit/api/models/repositories/test_template_repos.py index ffc6e343..57aba8e1 100644 --- a/tests/unit/api/models/repositories/test_template_repos.py +++ b/tests/unit/api/models/repositories/test_template_repos.py @@ -31,7 +31,7 @@ @pytest.fixture -def repo_info(simple_local_wf_repo) -> Dict[str, str]: +def repo_info() -> Dict[str, str]: return { "name": "MyWorkflowTest", "owner": "lm", From ff8d385da47f1479f4489131f6bd184382c2b8bc Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 4 Jul 2023 07:04:49 +0000 Subject: [PATCH 25/90] test: restore temporary path --- .../repositories/test_template_repos.py | 67 +++++++++---------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/tests/unit/api/models/repositories/test_template_repos.py b/tests/unit/api/models/repositories/test_template_repos.py index 57aba8e1..ba70f34c 100644 --- a/tests/unit/api/models/repositories/test_template_repos.py +++ b/tests/unit/api/models/repositories/test_template_repos.py @@ -56,37 +56,36 @@ def repo_template_type(request): def test_repo_template(repo_info, repo_template_type): - # with tempfile.TemporaryDirectory() as workflow_path: - workflow_path = tempfile.TemporaryDirectory().name - logger.debug("Creating a new Galaxy workflow repository template in %r", workflow_path) - # instantiate the template - tmpl = templates.WorkflowRepositoryTemplate.new_instance(repo_template_type, data={ - 'workflow_name': repo_info['name'], 'workflow_description': repo_info['description'], - 'workflow_version': '1.0.0', 'workflow_author': 'lm', 'workflow_license': repo_info['license'], - 'repo_url': repo_info['remote_url'], 'repo_full_name': repo_info['full_name'], - 'main_branch': repo_info['default_branch'] - }, local_path=workflow_path) - # check the template type - assert isinstance(tmpl, templates.WorkflowRepositoryTemplate), "Template is not a WorkflowRepositoryTemplate" - assert tmpl.type == repo_template_type, "Template type is not correct" - - # check if the template is the expected one - if repo_template_type == 'galaxy': - assert isinstance(tmpl, templates.galaxy.GalaxyRepositoryTemplate), "Template is not a GalaxyWorkflowTemplate" - if repo_template_type == 'nextflow': - assert isinstance(tmpl, templates.nextflow.NextflowRepositoryTemplate), "Template is not a SnakeMakeWorkflowTemplate" - if repo_template_type == 'snakemake': - assert isinstance(tmpl, templates.snakemake.SnakemakeRepositoryTemplate), "Template is not a NextflowWorkflowTemplate" - - # generate the repository - repo = tmpl.generate() - - # check the repository metadata - assert repo, "Repository object is None" - assert isinstance(repo, repos.LocalWorkflowRepository), "Repository is not a WorkflowRepository" - assert repo.name == repo_info['name'], "Repository name is not correct" - assert repo.owner == repo_info['owner'], "Repository owner is not correct" - assert repo.full_name == f"{repo_info['owner']}/{repo_info['name']}", "Repository full name is not correct" - assert repo.license == repo_info['license'], "Repository license is not correct" - assert repo.local_path == workflow_path, "Repository local path is not correct" - assert repo.remote_url == repo_info['remote_url'], "Repository remote url is not correct" + with tempfile.TemporaryDirectory() as workflow_path: + logger.debug("Creating a new Galaxy workflow repository template in %r", workflow_path) + # instantiate the template + tmpl = templates.WorkflowRepositoryTemplate.new_instance(repo_template_type, data={ + 'workflow_name': repo_info['name'], 'workflow_description': repo_info['description'], + 'workflow_version': '1.0.0', 'workflow_author': 'lm', 'workflow_license': repo_info['license'], + 'repo_url': repo_info['remote_url'], 'repo_full_name': repo_info['full_name'], + 'main_branch': repo_info['default_branch'] + }, local_path=workflow_path) + # check the template type + assert isinstance(tmpl, templates.WorkflowRepositoryTemplate), "Template is not a WorkflowRepositoryTemplate" + assert tmpl.type == repo_template_type, "Template type is not correct" + + # check if the template is the expected one + if repo_template_type == 'galaxy': + assert isinstance(tmpl, templates.galaxy.GalaxyRepositoryTemplate), "Template is not a GalaxyWorkflowTemplate" + if repo_template_type == 'nextflow': + assert isinstance(tmpl, templates.nextflow.NextflowRepositoryTemplate), "Template is not a SnakeMakeWorkflowTemplate" + if repo_template_type == 'snakemake': + assert isinstance(tmpl, templates.snakemake.SnakemakeRepositoryTemplate), "Template is not a NextflowWorkflowTemplate" + + # generate the repository + repo = tmpl.generate() + + # check the repository metadata + assert repo, "Repository object is None" + assert isinstance(repo, repos.LocalWorkflowRepository), "Repository is not a WorkflowRepository" + assert repo.name == repo_info['name'], "Repository name is not correct" + assert repo.owner == repo_info['owner'], "Repository owner is not correct" + assert repo.full_name == f"{repo_info['owner']}/{repo_info['name']}", "Repository full name is not correct" + assert repo.license == repo_info['license'], "Repository license is not correct" + assert repo.local_path == workflow_path, "Repository local path is not correct" + assert repo.remote_url == repo_info['remote_url'], "Repository remote url is not correct" From 83361ce105362973f900ba102ce63cb3d5aa1831 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 4 Jul 2023 08:31:55 +0000 Subject: [PATCH 26/90] test: update fixture --- tests/unit/api/models/repositories/test_template_repos.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/api/models/repositories/test_template_repos.py b/tests/unit/api/models/repositories/test_template_repos.py index ba70f34c..acad77d0 100644 --- a/tests/unit/api/models/repositories/test_template_repos.py +++ b/tests/unit/api/models/repositories/test_template_repos.py @@ -50,8 +50,8 @@ def repo_template_types() -> List[str]: @pytest.fixture(params=repo_template_types()) -def repo_template_type(request): - return request.param +def repo_template_type(template_param): + return template_param.param def test_repo_template(repo_info, repo_template_type): From 128d630063300c0d27650ab37912abfc970bfcd4 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 4 Jul 2023 09:08:46 +0000 Subject: [PATCH 27/90] test: remove parametric fixture --- .../repositories/test_template_repos.py | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/tests/unit/api/models/repositories/test_template_repos.py b/tests/unit/api/models/repositories/test_template_repos.py index acad77d0..5ba69ee0 100644 --- a/tests/unit/api/models/repositories/test_template_repos.py +++ b/tests/unit/api/models/repositories/test_template_repos.py @@ -49,43 +49,43 @@ def repo_template_types() -> List[str]: return ['galaxy', 'snakemake', 'nextflow', 'other'] -@pytest.fixture(params=repo_template_types()) -def repo_template_type(template_param): - return template_param.param - - -def test_repo_template(repo_info, repo_template_type): - - with tempfile.TemporaryDirectory() as workflow_path: - logger.debug("Creating a new Galaxy workflow repository template in %r", workflow_path) - # instantiate the template - tmpl = templates.WorkflowRepositoryTemplate.new_instance(repo_template_type, data={ - 'workflow_name': repo_info['name'], 'workflow_description': repo_info['description'], - 'workflow_version': '1.0.0', 'workflow_author': 'lm', 'workflow_license': repo_info['license'], - 'repo_url': repo_info['remote_url'], 'repo_full_name': repo_info['full_name'], - 'main_branch': repo_info['default_branch'] - }, local_path=workflow_path) - # check the template type - assert isinstance(tmpl, templates.WorkflowRepositoryTemplate), "Template is not a WorkflowRepositoryTemplate" - assert tmpl.type == repo_template_type, "Template type is not correct" - - # check if the template is the expected one - if repo_template_type == 'galaxy': - assert isinstance(tmpl, templates.galaxy.GalaxyRepositoryTemplate), "Template is not a GalaxyWorkflowTemplate" - if repo_template_type == 'nextflow': - assert isinstance(tmpl, templates.nextflow.NextflowRepositoryTemplate), "Template is not a SnakeMakeWorkflowTemplate" - if repo_template_type == 'snakemake': - assert isinstance(tmpl, templates.snakemake.SnakemakeRepositoryTemplate), "Template is not a NextflowWorkflowTemplate" - - # generate the repository - repo = tmpl.generate() - - # check the repository metadata - assert repo, "Repository object is None" - assert isinstance(repo, repos.LocalWorkflowRepository), "Repository is not a WorkflowRepository" - assert repo.name == repo_info['name'], "Repository name is not correct" - assert repo.owner == repo_info['owner'], "Repository owner is not correct" - assert repo.full_name == f"{repo_info['owner']}/{repo_info['name']}", "Repository full name is not correct" - assert repo.license == repo_info['license'], "Repository license is not correct" - assert repo.local_path == workflow_path, "Repository local path is not correct" - assert repo.remote_url == repo_info['remote_url'], "Repository remote url is not correct" +# @pytest.fixture(params=repo_template_types()) +# def repo_template_type(template_param): +# return template_param.param + + +def test_repo_template(repo_info): + for repo_template_type in repo_template_types(): + with tempfile.TemporaryDirectory() as workflow_path: + logger.debug("Creating a new Galaxy workflow repository template in %r", workflow_path) + # instantiate the template + tmpl = templates.WorkflowRepositoryTemplate.new_instance(repo_template_type, data={ + 'workflow_name': repo_info['name'], 'workflow_description': repo_info['description'], + 'workflow_version': '1.0.0', 'workflow_author': 'lm', 'workflow_license': repo_info['license'], + 'repo_url': repo_info['remote_url'], 'repo_full_name': repo_info['full_name'], + 'main_branch': repo_info['default_branch'] + }, local_path=workflow_path) + # check the template type + assert isinstance(tmpl, templates.WorkflowRepositoryTemplate), "Template is not a WorkflowRepositoryTemplate" + assert tmpl.type == repo_template_type, "Template type is not correct" + + # check if the template is the expected one + if repo_template_type == 'galaxy': + assert isinstance(tmpl, templates.galaxy.GalaxyRepositoryTemplate), "Template is not a GalaxyWorkflowTemplate" + if repo_template_type == 'nextflow': + assert isinstance(tmpl, templates.nextflow.NextflowRepositoryTemplate), "Template is not a SnakeMakeWorkflowTemplate" + if repo_template_type == 'snakemake': + assert isinstance(tmpl, templates.snakemake.SnakemakeRepositoryTemplate), "Template is not a NextflowWorkflowTemplate" + + # generate the repository + repo = tmpl.generate() + + # check the repository metadata + assert repo, "Repository object is None" + assert isinstance(repo, repos.LocalWorkflowRepository), "Repository is not a WorkflowRepository" + assert repo.name == repo_info['name'], "Repository name is not correct" + assert repo.owner == repo_info['owner'], "Repository owner is not correct" + assert repo.full_name == f"{repo_info['owner']}/{repo_info['name']}", "Repository full name is not correct" + assert repo.license == repo_info['license'], "Repository license is not correct" + assert repo.local_path == workflow_path, "Repository local path is not correct" + assert repo.remote_url == repo_info['remote_url'], "Repository remote url is not correct" From 7ab4077b50e83c08bc2d1174ab2d92c3643b945a Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 4 Jul 2023 09:56:26 +0000 Subject: [PATCH 28/90] add prefix --- tests/unit/api/models/repositories/test_template_repos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/api/models/repositories/test_template_repos.py b/tests/unit/api/models/repositories/test_template_repos.py index 5ba69ee0..85c0e1f7 100644 --- a/tests/unit/api/models/repositories/test_template_repos.py +++ b/tests/unit/api/models/repositories/test_template_repos.py @@ -56,7 +56,7 @@ def repo_template_types() -> List[str]: def test_repo_template(repo_info): for repo_template_type in repo_template_types(): - with tempfile.TemporaryDirectory() as workflow_path: + with tempfile.TemporaryDirectory(prefix=f"template-{repo_template_type}") as workflow_path: logger.debug("Creating a new Galaxy workflow repository template in %r", workflow_path) # instantiate the template tmpl = templates.WorkflowRepositoryTemplate.new_instance(repo_template_type, data={ From 6fc9d56533d2d8ff935cb72cc669a5afa8c67ba1 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 4 Jul 2023 10:40:32 +0000 Subject: [PATCH 29/90] update test --- .../repositories/test_template_repos.py | 73 +++++++++---------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/tests/unit/api/models/repositories/test_template_repos.py b/tests/unit/api/models/repositories/test_template_repos.py index 85c0e1f7..a4ff3015 100644 --- a/tests/unit/api/models/repositories/test_template_repos.py +++ b/tests/unit/api/models/repositories/test_template_repos.py @@ -24,8 +24,8 @@ import pytest -import lifemonitor.api.models.repositories as repos -import lifemonitor.api.models.repositories.templates as templates +from lifemonitor.api.models import repositories as repos +from lifemonitor.api.models.repositories import templates logger = logging.getLogger(__name__) @@ -49,43 +49,42 @@ def repo_template_types() -> List[str]: return ['galaxy', 'snakemake', 'nextflow', 'other'] -# @pytest.fixture(params=repo_template_types()) -# def repo_template_type(template_param): -# return template_param.param +@pytest.fixture(params=repo_template_types()) +def repo_template_type(request): + return request.param -def test_repo_template(repo_info): - for repo_template_type in repo_template_types(): - with tempfile.TemporaryDirectory(prefix=f"template-{repo_template_type}") as workflow_path: - logger.debug("Creating a new Galaxy workflow repository template in %r", workflow_path) - # instantiate the template - tmpl = templates.WorkflowRepositoryTemplate.new_instance(repo_template_type, data={ - 'workflow_name': repo_info['name'], 'workflow_description': repo_info['description'], - 'workflow_version': '1.0.0', 'workflow_author': 'lm', 'workflow_license': repo_info['license'], - 'repo_url': repo_info['remote_url'], 'repo_full_name': repo_info['full_name'], - 'main_branch': repo_info['default_branch'] - }, local_path=workflow_path) - # check the template type - assert isinstance(tmpl, templates.WorkflowRepositoryTemplate), "Template is not a WorkflowRepositoryTemplate" - assert tmpl.type == repo_template_type, "Template type is not correct" +def test_repo_template(repo_info, repo_template_type): + with tempfile.TemporaryDirectory(prefix=f"template-{repo_template_type}") as workflow_path: + logger.debug("Creating a new Galaxy workflow repository template in %r", workflow_path) + # instantiate the template + tmpl = templates.WorkflowRepositoryTemplate.new_instance(repo_template_type, data={ + 'workflow_name': repo_info['name'], 'workflow_description': repo_info['description'], + 'workflow_version': '1.0.0', 'workflow_author': 'lm', 'workflow_license': repo_info['license'], + 'repo_url': repo_info['remote_url'], 'repo_full_name': repo_info['full_name'], + 'main_branch': repo_info['default_branch'] + }, local_path=workflow_path) + # check the template type + assert isinstance(tmpl, templates.WorkflowRepositoryTemplate), "Template is not a WorkflowRepositoryTemplate" + assert tmpl.type == repo_template_type, "Template type is not correct" - # check if the template is the expected one - if repo_template_type == 'galaxy': - assert isinstance(tmpl, templates.galaxy.GalaxyRepositoryTemplate), "Template is not a GalaxyWorkflowTemplate" - if repo_template_type == 'nextflow': - assert isinstance(tmpl, templates.nextflow.NextflowRepositoryTemplate), "Template is not a SnakeMakeWorkflowTemplate" - if repo_template_type == 'snakemake': - assert isinstance(tmpl, templates.snakemake.SnakemakeRepositoryTemplate), "Template is not a NextflowWorkflowTemplate" + # check if the template is the expected one + if repo_template_type == 'galaxy': + assert isinstance(tmpl, templates.galaxy.GalaxyRepositoryTemplate), "Template is not a GalaxyWorkflowTemplate" + if repo_template_type == 'nextflow': + assert isinstance(tmpl, templates.nextflow.NextflowRepositoryTemplate), "Template is not a SnakeMakeWorkflowTemplate" + if repo_template_type == 'snakemake': + assert isinstance(tmpl, templates.snakemake.SnakemakeRepositoryTemplate), "Template is not a NextflowWorkflowTemplate" - # generate the repository - repo = tmpl.generate() + # generate the repository + repo = tmpl.generate() - # check the repository metadata - assert repo, "Repository object is None" - assert isinstance(repo, repos.LocalWorkflowRepository), "Repository is not a WorkflowRepository" - assert repo.name == repo_info['name'], "Repository name is not correct" - assert repo.owner == repo_info['owner'], "Repository owner is not correct" - assert repo.full_name == f"{repo_info['owner']}/{repo_info['name']}", "Repository full name is not correct" - assert repo.license == repo_info['license'], "Repository license is not correct" - assert repo.local_path == workflow_path, "Repository local path is not correct" - assert repo.remote_url == repo_info['remote_url'], "Repository remote url is not correct" + # check the repository metadata + assert repo, "Repository object is None" + assert isinstance(repo, repos.LocalWorkflowRepository), "Repository is not a WorkflowRepository" + assert repo.name == repo_info['name'], "Repository name is not correct" + assert repo.owner == repo_info['owner'], "Repository owner is not correct" + assert repo.full_name == f"{repo_info['owner']}/{repo_info['name']}", "Repository full name is not correct" + assert repo.license == repo_info['license'], "Repository license is not correct" + assert repo.local_path == workflow_path, "Repository local path is not correct" + assert repo.remote_url == repo_info['remote_url'], "Repository remote url is not correct" From b1a1c7cf99c35cabd77a416d0ee3cd7d44d9b773 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 4 Jul 2023 11:01:29 +0000 Subject: [PATCH 30/90] order dependencies test: use user1 fixture isolate workflow templates skip tests --- Makefile | 2 +- tests/unit/api/models/repositories/test_template_repos.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 2fe49d89..dddd13a7 100644 --- a/Makefile +++ b/Makefile @@ -269,7 +269,7 @@ run-tests: start-testing ## Run all tests in the Testing Environment tests: start-testing ## CI utility to setup, run tests and teardown a testing environment @printf "\n$(bold)Running tests...$(reset)\n" ; \ $(docker_compose) -f ./docker-compose.yml \ - exec -T lmtests /bin/bash -c "pytest --reruns 2 --reruns-delay 5 --durations=10 --color=yes tests"; \ + exec -T lmtests /bin/bash -c "pytest --reruns 2 --reruns-delay 5 --durations=10 --color=yes tests --order-dependencies"; \ result=$$?; \ printf "\n$(bold)Teardown services...$(reset)\n" ; \ USER_UID=$$(id -u) USER_GID=$$(id -g) \ diff --git a/tests/unit/api/models/repositories/test_template_repos.py b/tests/unit/api/models/repositories/test_template_repos.py index a4ff3015..78fa3719 100644 --- a/tests/unit/api/models/repositories/test_template_repos.py +++ b/tests/unit/api/models/repositories/test_template_repos.py @@ -46,7 +46,7 @@ def repo_info() -> Dict[str, str]: def repo_template_types() -> List[str]: - return ['galaxy', 'snakemake', 'nextflow', 'other'] + return ['galaxy', 'snakemake'] # , 'nextflow', 'other'] @pytest.fixture(params=repo_template_types()) @@ -54,7 +54,8 @@ def repo_template_type(request): return request.param -def test_repo_template(repo_info, repo_template_type): +@pytest.mark.skip(reason="Template type not supported") +def test_repo_template(user1, repo_info, repo_template_type): with tempfile.TemporaryDirectory(prefix=f"template-{repo_template_type}") as workflow_path: logger.debug("Creating a new Galaxy workflow repository template in %r", workflow_path) # instantiate the template From 47d03593dfd12c8ad99e7be5c076d5b81fde73a2 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 4 Jul 2023 16:38:17 +0000 Subject: [PATCH 31/90] rename tests --- .../{test_template_repos.py => test_base_templates.py} | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) rename tests/unit/api/models/repositories/{test_template_repos.py => test_base_templates.py} (97%) diff --git a/tests/unit/api/models/repositories/test_template_repos.py b/tests/unit/api/models/repositories/test_base_templates.py similarity index 97% rename from tests/unit/api/models/repositories/test_template_repos.py rename to tests/unit/api/models/repositories/test_base_templates.py index 78fa3719..f633f1d3 100644 --- a/tests/unit/api/models/repositories/test_template_repos.py +++ b/tests/unit/api/models/repositories/test_base_templates.py @@ -46,7 +46,7 @@ def repo_info() -> Dict[str, str]: def repo_template_types() -> List[str]: - return ['galaxy', 'snakemake'] # , 'nextflow', 'other'] + return ['galaxy', 'snakemake', 'nextflow', 'other'] @pytest.fixture(params=repo_template_types()) @@ -54,7 +54,6 @@ def repo_template_type(request): return request.param -@pytest.mark.skip(reason="Template type not supported") def test_repo_template(user1, repo_info, repo_template_type): with tempfile.TemporaryDirectory(prefix=f"template-{repo_template_type}") as workflow_path: logger.debug("Creating a new Galaxy workflow repository template in %r", workflow_path) From 9613a30606fbbd83b8c56eb014375c5b38eb93bb Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 5 Jul 2023 08:21:51 +0000 Subject: [PATCH 32/90] fix: close sessions --- tests/conftest_helpers.py | 17 +++++++++++------ tests/test_users.py | 28 ++++++++++++++-------------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/tests/conftest_helpers.py b/tests/conftest_helpers.py index a599e8a8..3d31ba9a 100644 --- a/tests/conftest_helpers.py +++ b/tests/conftest_helpers.py @@ -280,12 +280,17 @@ def user(_app_context, _provider_type, _user_index=1, _register_workflows=False) } if _register_workflows: utils.register_workflows(user_obj) - yield user_obj - if user and not user.is_anonymous: - try: - logout_user() - except Exception: - pass + try: + yield user_obj + except Exception as e: + if logger.isEnabledFor(logging.DEBUG): + logger.exception(e) + finally: + if user and not user.is_anonymous: + try: + logout_user() + except Exception: + pass except KeyError as e: logger.exception(e) raise RuntimeError(f"Authorization provider {_provider_type} is not supported") diff --git a/tests/test_users.py b/tests/test_users.py index 9900ee08..eb75384c 100644 --- a/tests/test_users.py +++ b/tests/test_users.py @@ -78,19 +78,19 @@ def test_user1_auth(user1, client_auth_method, user1_auth): def test_user_auto_logout(user1, client_auth_method, user1_auth): logger.debug("Auth: %r, %r, %r", user1_auth, client_auth_method, ClientAuthenticationMethod.BASIC.value) - app_client = requests.session() - app_client_url = f'{get_base_url()}/users/current' - logger.debug("client URL: %r", app_client_url) + with requests.Session() as app_client: + app_client_url = f'{get_base_url()}/users/current' + logger.debug("client URL: %r", app_client_url) - r1 = app_client.get(app_client_url, headers=user1_auth) - logger.debug("headers: %r", r1.headers) - logger.debug("response: %r", r1.content) - if client_auth_method in [ClientAuthenticationMethod.NOAUTH, ClientAuthenticationMethod.BASIC]: - assert r1.status_code == 401, "Expected 401 status code" - else: - assert r1.status_code == 200, "Expected 200 status code" - logger.debug("Response R1: %r", r1.json) + r1 = app_client.get(app_client_url, headers=user1_auth) + logger.debug("headers: %r", r1.headers) + logger.debug("response: %r", r1.content) + if client_auth_method in [ClientAuthenticationMethod.NOAUTH, ClientAuthenticationMethod.BASIC]: + assert r1.status_code == 401, "Expected 401 status code" + else: + assert r1.status_code == 200, "Expected 200 status code" + logger.debug("Response R1: %r", r1.json) - r2 = app_client.get(app_client_url) - logger.debug("headers: %r", r2.headers) - assert r2.status_code == 401, "Expected 401 status code" + r2 = app_client.get(app_client_url) + logger.debug("headers: %r", r2.headers) + assert r2.status_code == 401, "Expected 401 status code" From 93772a6236a07c6827791ebea85c3523968999a3 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 5 Jul 2023 09:14:18 +0000 Subject: [PATCH 33/90] fix: delete connected local repo --- lifemonitor/api/models/repositories/github.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lifemonitor/api/models/repositories/github.py b/lifemonitor/api/models/repositories/github.py index 71ef28cb..a223e89d 100644 --- a/lifemonitor/api/models/repositories/github.py +++ b/lifemonitor/api/models/repositories/github.py @@ -381,9 +381,11 @@ def __del__(self): def cleanup(self) -> None: logger.debug("Repository cleanup") - if self._local_repo: + if getattr(self, "_local_repo", None): + local_repo_path = self.local_repo.local_path + del self._local_repo logger.debug("Removing temp folder %r of %r", self.local_path, self) - shutil.rmtree(self.local_repo.local_path, ignore_errors=True) + shutil.rmtree(local_repo_path, ignore_errors=True) self._local_repo = None From cc542dc3e2e91b138d1433fe502c92904095eac0 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 5 Jul 2023 09:18:24 +0000 Subject: [PATCH 34/90] build(pip): update pytest --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 379e600f..8388cab7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,9 +31,9 @@ prometheus-flask-exporter~=0.22 pygit2~=1.12.1 psycopg2-binary~=2.9.6 pyopenssl==23.2.0 -pytest-mock~=3.10.0 -pytest~=7.3.1 -pytest-rerunfailures~=11.1.2 +pytest-mock~=3.11.1 +pytest~=7.4.0 +pytest-rerunfailures~=12.0.0 pytest-xdist~=3.3.1 python-dotenv~=0.19.0 python-jenkins==1.7.0 From 7c66a4f3f34493c66128c6fd3b93d53b57ae7e5a Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 6 Jul 2023 16:53:11 +0000 Subject: [PATCH 35/90] fix(model): revert to the right logic --- lifemonitor/api/models/repositories/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lifemonitor/api/models/repositories/github.py b/lifemonitor/api/models/repositories/github.py index a223e89d..6d88496d 100644 --- a/lifemonitor/api/models/repositories/github.py +++ b/lifemonitor/api/models/repositories/github.py @@ -361,7 +361,7 @@ def write_zip(self, target_path: str): @property def local_repo(self) -> LocalGitWorkflowRepository: - if not self._local_repo: + if not getattr(self, "_local_repo", None): local_path = self._local_path or tempfile.mkdtemp(dir=BaseConfig.BASE_TEMP_FOLDER) if not os.path.exists(local_path) or not LocalWorkflowRepository.is_git_repo(local_path): logger.debug("Cloning %r", self.clone_url) From c8000474a2670e9f4a010163302a7ec55da23af9 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 7 Jul 2023 05:47:03 +0000 Subject: [PATCH 36/90] fix: remove unused fixtures --- tests/unit/auth/models/test_tokens.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/auth/models/test_tokens.py b/tests/unit/auth/models/test_tokens.py index a0fc0968..fddedcd4 100644 --- a/tests/unit/auth/models/test_tokens.py +++ b/tests/unit/auth/models/test_tokens.py @@ -35,7 +35,7 @@ @pytest.fixture -def user_identity(app_client, user1, provider_type): +def user_identity(user1, provider_type): logger.debug(user1.keys()) logger.debug(user1['user_info'].keys()) user = user1['user'] From ffbe3289f27d88c8c0b457788f810e9477e45cdb Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 7 Jul 2023 07:19:18 +0000 Subject: [PATCH 37/90] refactor(model): remove problematic imports. In particular, nf-core imported at module level causes several issues when running tests. --- .../models/repositories/templates/nextflow.py | 52 +++++-------------- 1 file changed, 12 insertions(+), 40 deletions(-) diff --git a/lifemonitor/api/models/repositories/templates/nextflow.py b/lifemonitor/api/models/repositories/templates/nextflow.py index 9574548a..a21a3877 100644 --- a/lifemonitor/api/models/repositories/templates/nextflow.py +++ b/lifemonitor/api/models/repositories/templates/nextflow.py @@ -25,18 +25,14 @@ import re from typing import Dict, List, Optional -import nf_core.create -from lifemonitor.api.models.repositories.files.base import RepositoryFile -from lifemonitor.api.models.repositories.local import LocalWorkflowRepository -from nf_core import utils as nf_core_utils +import lifemonitor.api.models.repositories as repos +import lifemonitor.api.models.repositories.files as repo_files from . import WorkflowRepositoryTemplate # set module level logger logger = logging.getLogger(__name__) -# log loaded nf_core.utils is loaded -logger.debug("Loaded module 'nf-core.utils': %r", nf_core_utils) # Ignore LICENSE ignore_license = """ @@ -59,27 +55,29 @@ def __init__(self, data: Optional[Dict[str, str]] = None, super().__init__(data=data, local_path=local_path, init_git=init_git) @property - def files(self) -> List[RepositoryFile]: + def files(self) -> List[repo_files.base.RepositoryFile]: result = [] # skip = self._transient_files['remove'].keys() for root, _, files in os.path.walk(self.local_path, exclude=self.exclude): dirname = root.replace(self.local_path, '.') for name in files: # if f"{dirname}/{name}" not in skip: - result.append(RepositoryFile(self.local_path, name, dir=dirname)) + result.append(repo_files.base.RepositoryFile(self.local_path, name, dir=dirname)) result.extend([v for k, v in self._transient_files['add'].items()]) return result - def generate(self, target_path: str = None) -> LocalWorkflowRepository: + def generate(self, target_path: str = None) -> repos.LocalWorkflowRepository: target_path = target_path or self.local_path logger.debug("Rendering template files to %s...", target_path) # name, description, author, version="1.0dev", no_git=False, force=False, outdir=None - create_obj = NextflowPipeline( - self.data.get("workflow_name"), - self.data.get("workflow_description", ""), + from nf_core.create import PipelineCreate + create_obj = PipelineCreate( + re.sub(r"\s+", "", self.data.get("workflow_name")), + self.data.get("workflow_description"), self.data.get("workflow_author", ""), - self.data.get('workflow_version', "0.1.0"), - False, True, outdir=target_path, plain=True) + self.data.get("workflow_version", "0.1.0"), + False, True, outdir=target_path, plain=True + ) create_obj.init_pipeline() # patch prettier config to ignore crate and lm metadata @@ -104,29 +102,3 @@ def generate(self, target_path: str = None) -> LocalWorkflowRepository: repo.generate_metadata(**self.data) # return the repository object return repo - - -class NextflowPipeline(nf_core.create.PipelineCreate): - - def __init__(self, name, description, author, version="1.0dev", no_git=False, - force=False, outdir=None, template_yaml=None, plain=True): - """ Override default constructor to properly set workflow name""" - super().__init__(re.sub(r"\s+", "", name), description, author, version, no_git, - force, outdir, template_yaml_path=template_yaml, plain=plain) - - # def git_init_pipeline(self): - # """Initialises the new pipeline as a Git repository and submits first commit.""" - # logger.info("Initialising pipeline git repository") - # repo = git.Repo.init(self.outdir) - # repo.git.add(A=True) - # repo.index.commit(f"initial template build from nf-core/tools, version {nf_core.__version__}") - # # Add TEMPLATE branch to git repository - # repo.git.branch("TEMPLATE") - # repo.git.branch("dev") - # logger.info( - # "Done. Remember to add a remote and push to GitHub:\n" - # f"[white on grey23] cd {self.outdir} \n" - # " git remote add origin git@github.com:USERNAME/REPO_NAME.git \n" - # " git push --all origin " - # ) - # logger.info("This will also push your newly created dev branch and the TEMPLATE branch for syncing.") From 1945472c8c757e2e4c9c65f4544ec158b1c1ef15 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 7 Jul 2023 07:19:54 +0000 Subject: [PATCH 38/90] refactor: restore original test filename --- .../repositories/test_base_templates.py | 90 ----------------- .../repositories/test_repo_templates.py | 99 +++++++++++++++++++ 2 files changed, 99 insertions(+), 90 deletions(-) delete mode 100644 tests/unit/api/models/repositories/test_base_templates.py create mode 100644 tests/unit/api/models/repositories/test_repo_templates.py diff --git a/tests/unit/api/models/repositories/test_base_templates.py b/tests/unit/api/models/repositories/test_base_templates.py deleted file mode 100644 index f633f1d3..00000000 --- a/tests/unit/api/models/repositories/test_base_templates.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright (c) 2020-2022 CRS4 -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import logging -import tempfile -from typing import Dict, List - -import pytest - -from lifemonitor.api.models import repositories as repos -from lifemonitor.api.models.repositories import templates - -logger = logging.getLogger(__name__) - - -@pytest.fixture -def repo_info() -> Dict[str, str]: - return { - "name": "MyWorkflowTest", - "owner": "lm", - "description": "My workflow test description", - "license": "MIT", - "exclude": [".*"], - "default_branch": "main", - "active_branch": "main", - "full_name": "lm/MyWorkflowTest", - "remote_url": 'https://github.com/lm/MyWorkflowTest', - } - - -def repo_template_types() -> List[str]: - return ['galaxy', 'snakemake', 'nextflow', 'other'] - - -@pytest.fixture(params=repo_template_types()) -def repo_template_type(request): - return request.param - - -def test_repo_template(user1, repo_info, repo_template_type): - with tempfile.TemporaryDirectory(prefix=f"template-{repo_template_type}") as workflow_path: - logger.debug("Creating a new Galaxy workflow repository template in %r", workflow_path) - # instantiate the template - tmpl = templates.WorkflowRepositoryTemplate.new_instance(repo_template_type, data={ - 'workflow_name': repo_info['name'], 'workflow_description': repo_info['description'], - 'workflow_version': '1.0.0', 'workflow_author': 'lm', 'workflow_license': repo_info['license'], - 'repo_url': repo_info['remote_url'], 'repo_full_name': repo_info['full_name'], - 'main_branch': repo_info['default_branch'] - }, local_path=workflow_path) - # check the template type - assert isinstance(tmpl, templates.WorkflowRepositoryTemplate), "Template is not a WorkflowRepositoryTemplate" - assert tmpl.type == repo_template_type, "Template type is not correct" - - # check if the template is the expected one - if repo_template_type == 'galaxy': - assert isinstance(tmpl, templates.galaxy.GalaxyRepositoryTemplate), "Template is not a GalaxyWorkflowTemplate" - if repo_template_type == 'nextflow': - assert isinstance(tmpl, templates.nextflow.NextflowRepositoryTemplate), "Template is not a SnakeMakeWorkflowTemplate" - if repo_template_type == 'snakemake': - assert isinstance(tmpl, templates.snakemake.SnakemakeRepositoryTemplate), "Template is not a NextflowWorkflowTemplate" - - # generate the repository - repo = tmpl.generate() - - # check the repository metadata - assert repo, "Repository object is None" - assert isinstance(repo, repos.LocalWorkflowRepository), "Repository is not a WorkflowRepository" - assert repo.name == repo_info['name'], "Repository name is not correct" - assert repo.owner == repo_info['owner'], "Repository owner is not correct" - assert repo.full_name == f"{repo_info['owner']}/{repo_info['name']}", "Repository full name is not correct" - assert repo.license == repo_info['license'], "Repository license is not correct" - assert repo.local_path == workflow_path, "Repository local path is not correct" - assert repo.remote_url == repo_info['remote_url'], "Repository remote url is not correct" diff --git a/tests/unit/api/models/repositories/test_repo_templates.py b/tests/unit/api/models/repositories/test_repo_templates.py new file mode 100644 index 00000000..338af6a1 --- /dev/null +++ b/tests/unit/api/models/repositories/test_repo_templates.py @@ -0,0 +1,99 @@ +# Copyright (c) 2020-2022 CRS4 +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import logging +import tempfile +from typing import Dict, List + +import pytest + +from lifemonitor.api.models import repositories as repos +from lifemonitor.api.models.repositories import templates + +logger = logging.getLogger(__name__) + + +@pytest.fixture +def repository_info() -> Dict[str, str]: + return { + "name": "MyWorkflowTest", + "owner": "lm", + "description": "My workflow test description", + "license": "MIT", + "exclude": [".*"], + "default_branch": "main", + "active_branch": "main", + "full_name": "lm/MyWorkflowTest", + "remote_url": 'https://github.com/lm/MyWorkflowTest', + } + + +def repo_template_types() -> List[str]: + return ['galaxy', 'snakemake', 'nextflow', 'other'] + + +@pytest.mark.parametrize("repo_template_type", repo_template_types()) +def test_repo_template(repository_info, repo_template_type): + # for repo_template_type in repo_template_types(): + with tempfile.TemporaryDirectory(prefix=f"template-{repo_template_type}") as workflow_path: + logger.debug("Creating a new Galaxy workflow repository template in %r", workflow_path) + # instantiate the template + from lifemonitor.api.models.repositories.templates.galaxy import GalaxyRepositoryTemplate + from lifemonitor.api.models.repositories.templates import WorkflowRepositoryTemplate + from lifemonitor.api.models.repositories.templates.nextflow import NextflowRepositoryTemplate + from lifemonitor.api.models.repositories.templates.snakemake import SnakemakeRepositoryTemplate + tmpl = GalaxyRepositoryTemplate( + data={ + 'workflow_name': repository_info['name'], 'workflow_description': repository_info['description'], + 'workflow_version': '1.0.0', 'workflow_author': 'lm', 'workflow_license': repository_info['license'], + 'repo_url': repository_info['remote_url'], 'repo_full_name': repository_info['full_name'], + 'main_branch': repository_info['default_branch'] + }, local_path=workflow_path + ) + # tmpl = templates.WorkflowRepositoryTemplate.new_instance(repo_template_type, data={ + # 'workflow_name': repository_info['name'], 'workflow_description': repository_info['description'], + # 'workflow_version': '1.0.0', 'workflow_author': 'lm', 'workflow_license': repository_info['license'], + # 'repo_url': repository_info['remote_url'], 'repo_full_name': repository_info['full_name'], + # 'main_branch': repository_info['default_branch'] + # }, local_path=workflow_path) + # # # check the template type + # assert isinstance(tmpl, templates.WorkflowRepositoryTemplate), "Template is not a WorkflowRepositoryTemplate" + # assert tmpl.type == repo_template_type, "Template type is not correct" + + # # check if the template is the expected one + # if repo_template_type == 'galaxy': + # assert isinstance(tmpl, templates.galaxy.GalaxyRepositoryTemplate), "Template is not a GalaxyWorkflowTemplate" + # if repo_template_type == 'nextflow': + # assert isinstance(tmpl, templates.nextflow.NextflowRepositoryTemplate), "Template is not a SnakeMakeWorkflowTemplate" + # if repo_template_type == 'snakemake': + # assert isinstance(tmpl, templates.snakemake.SnakemakeRepositoryTemplate), "Template is not a NextflowWorkflowTemplate" + + # # generate the repository + # repo = tmpl.generate() + + # # check the repository metadata + # assert repo, "Repository object is None" + # assert isinstance(repo, repos.LocalWorkflowRepository), "Repository is not a WorkflowRepository" + # assert repo.name == repository_info['name'], "Repository name is not correct" + # assert repo.owner == repository_info['owner'], "Repository owner is not correct" + # assert repo.full_name == f"{repository_info['owner']}/{repository_info['name']}", "Repository full name is not correct" + # assert repo.license == repository_info['license'], "Repository license is not correct" + # assert repo.local_path == workflow_path, "Repository local path is not correct" + # assert repo.remote_url == repository_info['remote_url'], "Repository remote url is not correct" From 977450cbea4e6e83307ef49be55989ac1f525568 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 7 Jul 2023 07:22:10 +0000 Subject: [PATCH 39/90] refactor: revert original test logic --- .../repositories/test_repo_templates.py | 66 ++++++++----------- 1 file changed, 27 insertions(+), 39 deletions(-) diff --git a/tests/unit/api/models/repositories/test_repo_templates.py b/tests/unit/api/models/repositories/test_repo_templates.py index 338af6a1..5d7ca937 100644 --- a/tests/unit/api/models/repositories/test_repo_templates.py +++ b/tests/unit/api/models/repositories/test_repo_templates.py @@ -55,45 +55,33 @@ def test_repo_template(repository_info, repo_template_type): with tempfile.TemporaryDirectory(prefix=f"template-{repo_template_type}") as workflow_path: logger.debug("Creating a new Galaxy workflow repository template in %r", workflow_path) # instantiate the template - from lifemonitor.api.models.repositories.templates.galaxy import GalaxyRepositoryTemplate - from lifemonitor.api.models.repositories.templates import WorkflowRepositoryTemplate - from lifemonitor.api.models.repositories.templates.nextflow import NextflowRepositoryTemplate - from lifemonitor.api.models.repositories.templates.snakemake import SnakemakeRepositoryTemplate - tmpl = GalaxyRepositoryTemplate( - data={ - 'workflow_name': repository_info['name'], 'workflow_description': repository_info['description'], - 'workflow_version': '1.0.0', 'workflow_author': 'lm', 'workflow_license': repository_info['license'], - 'repo_url': repository_info['remote_url'], 'repo_full_name': repository_info['full_name'], - 'main_branch': repository_info['default_branch'] - }, local_path=workflow_path - ) - # tmpl = templates.WorkflowRepositoryTemplate.new_instance(repo_template_type, data={ - # 'workflow_name': repository_info['name'], 'workflow_description': repository_info['description'], - # 'workflow_version': '1.0.0', 'workflow_author': 'lm', 'workflow_license': repository_info['license'], - # 'repo_url': repository_info['remote_url'], 'repo_full_name': repository_info['full_name'], - # 'main_branch': repository_info['default_branch'] - # }, local_path=workflow_path) - # # # check the template type - # assert isinstance(tmpl, templates.WorkflowRepositoryTemplate), "Template is not a WorkflowRepositoryTemplate" - # assert tmpl.type == repo_template_type, "Template type is not correct" + tmpl = templates.WorkflowRepositoryTemplate.new_instance(repo_template_type, data={ + 'workflow_name': repository_info['name'], 'workflow_description': repository_info['description'], + 'workflow_version': '1.0.0', 'workflow_author': 'lm', 'workflow_license': repository_info['license'], + 'repo_url': repository_info['remote_url'], 'repo_full_name': repository_info['full_name'], + 'main_branch': repository_info['default_branch'] + }, local_path=workflow_path) + # # check the template type + assert isinstance(tmpl, templates.WorkflowRepositoryTemplate), "Template is not a WorkflowRepositoryTemplate" + assert tmpl.type == repo_template_type, "Template type is not correct" - # # check if the template is the expected one - # if repo_template_type == 'galaxy': - # assert isinstance(tmpl, templates.galaxy.GalaxyRepositoryTemplate), "Template is not a GalaxyWorkflowTemplate" - # if repo_template_type == 'nextflow': - # assert isinstance(tmpl, templates.nextflow.NextflowRepositoryTemplate), "Template is not a SnakeMakeWorkflowTemplate" - # if repo_template_type == 'snakemake': - # assert isinstance(tmpl, templates.snakemake.SnakemakeRepositoryTemplate), "Template is not a NextflowWorkflowTemplate" + # check if the template is the expected one + if repo_template_type == 'galaxy': + assert isinstance(tmpl, templates.galaxy.GalaxyRepositoryTemplate), "Template is not a GalaxyWorkflowTemplate" + if repo_template_type == 'nextflow': + assert isinstance(tmpl, templates.nextflow.NextflowRepositoryTemplate), "Template is not a SnakeMakeWorkflowTemplate" + if repo_template_type == 'snakemake': + assert isinstance(tmpl, templates.snakemake.SnakemakeRepositoryTemplate), "Template is not a NextflowWorkflowTemplate" - # # generate the repository - # repo = tmpl.generate() + # generate the repository + repo = tmpl.generate() - # # check the repository metadata - # assert repo, "Repository object is None" - # assert isinstance(repo, repos.LocalWorkflowRepository), "Repository is not a WorkflowRepository" - # assert repo.name == repository_info['name'], "Repository name is not correct" - # assert repo.owner == repository_info['owner'], "Repository owner is not correct" - # assert repo.full_name == f"{repository_info['owner']}/{repository_info['name']}", "Repository full name is not correct" - # assert repo.license == repository_info['license'], "Repository license is not correct" - # assert repo.local_path == workflow_path, "Repository local path is not correct" - # assert repo.remote_url == repository_info['remote_url'], "Repository remote url is not correct" + # check the repository metadata + assert repo, "Repository object is None" + assert isinstance(repo, repos.LocalWorkflowRepository), "Repository is not a WorkflowRepository" + assert repo.name == repository_info['name'], "Repository name is not correct" + assert repo.owner == repository_info['owner'], "Repository owner is not correct" + assert repo.full_name == f"{repository_info['owner']}/{repository_info['name']}", "Repository full name is not correct" + assert repo.license == repository_info['license'], "Repository license is not correct" + assert repo.local_path == workflow_path, "Repository local path is not correct" + assert repo.remote_url == repository_info['remote_url'], "Repository remote url is not correct" From c74d09070ec2bfda323747f8d61c4e85214203a3 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 7 Jul 2023 08:06:11 +0000 Subject: [PATCH 40/90] test: skip test of snakemake template repo --- tests/unit/api/models/repositories/test_repo_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/api/models/repositories/test_repo_templates.py b/tests/unit/api/models/repositories/test_repo_templates.py index 5d7ca937..e5848615 100644 --- a/tests/unit/api/models/repositories/test_repo_templates.py +++ b/tests/unit/api/models/repositories/test_repo_templates.py @@ -46,7 +46,7 @@ def repository_info() -> Dict[str, str]: def repo_template_types() -> List[str]: - return ['galaxy', 'snakemake', 'nextflow', 'other'] + return ['galaxy', 'nextflow', 'other'] @pytest.mark.parametrize("repo_template_type", repo_template_types()) From 1e1ed1dd891e2eee19ad679f359f2addbf6cb4b0 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 7 Jul 2023 09:53:34 +0000 Subject: [PATCH 41/90] test: allow to skip all tests of repo templates --- .../unit/api/models/repositories/test_repo_templates.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/unit/api/models/repositories/test_repo_templates.py b/tests/unit/api/models/repositories/test_repo_templates.py index e5848615..38307438 100644 --- a/tests/unit/api/models/repositories/test_repo_templates.py +++ b/tests/unit/api/models/repositories/test_repo_templates.py @@ -46,12 +46,17 @@ def repository_info() -> Dict[str, str]: def repo_template_types() -> List[str]: - return ['galaxy', 'nextflow', 'other'] + types = ['galaxy', 'snakemake', 'nextflow', 'other'] + skip_types = os.environ.get('TEST_SKIP_TEMPLATE_TYPES', None) + for skip_type in skip_types.split(',') if skip_types else []: + types.remove(skip_type) + logger.debug("Skipping the following template types: %r", types) + return types +@pytest.mark.skipif(boolean_value(os.environ.get('TEST_SKIP_REPO_TEMPLATES', None)) is True, reason="Skipping repo templates") @pytest.mark.parametrize("repo_template_type", repo_template_types()) def test_repo_template(repository_info, repo_template_type): - # for repo_template_type in repo_template_types(): with tempfile.TemporaryDirectory(prefix=f"template-{repo_template_type}") as workflow_path: logger.debug("Creating a new Galaxy workflow repository template in %r", workflow_path) # instantiate the template From 62ae43fa2ae20e806e216a0e68e6ff73619ee348 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 7 Jul 2023 09:59:13 +0000 Subject: [PATCH 42/90] test: skip tests for repo templates by default --- tests/unit/api/models/repositories/test_repo_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/api/models/repositories/test_repo_templates.py b/tests/unit/api/models/repositories/test_repo_templates.py index 38307438..3cbd135f 100644 --- a/tests/unit/api/models/repositories/test_repo_templates.py +++ b/tests/unit/api/models/repositories/test_repo_templates.py @@ -54,7 +54,7 @@ def repo_template_types() -> List[str]: return types -@pytest.mark.skipif(boolean_value(os.environ.get('TEST_SKIP_REPO_TEMPLATES', None)) is True, reason="Skipping repo templates") +@pytest.mark.skipif(boolean_value(os.environ.get('TEST_SKIP_REPO_TEMPLATES', True)) is True, reason="Skipping repo templates") @pytest.mark.parametrize("repo_template_type", repo_template_types()) def test_repo_template(repository_info, repo_template_type): with tempfile.TemporaryDirectory(prefix=f"template-{repo_template_type}") as workflow_path: From 1c460568ef8026f99fb52a32e4c94e6ab6631f8e Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 7 Jul 2023 10:00:17 +0000 Subject: [PATCH 43/90] test: check list of files automatically generated by templates --- tests/unit/api/models/repositories/test_repo_templates.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/api/models/repositories/test_repo_templates.py b/tests/unit/api/models/repositories/test_repo_templates.py index 3cbd135f..84652ec0 100644 --- a/tests/unit/api/models/repositories/test_repo_templates.py +++ b/tests/unit/api/models/repositories/test_repo_templates.py @@ -90,3 +90,7 @@ def test_repo_template(repository_info, repo_template_type): assert repo.license == repository_info['license'], "Repository license is not correct" assert repo.local_path == workflow_path, "Repository local path is not correct" assert repo.remote_url == repository_info['remote_url'], "Repository remote url is not correct" + + # check the repository files + assert os.path.exists(os.path.join(workflow_path, 'README.md')), "README.md file does not exist" + assert os.path.exists(os.path.join(workflow_path, 'ro-crate-metadata.json')), "ro-crate-metadata.json file does not exist" From 5b38d207ba1cc363149fe4120ecd3c168971156f Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 7 Jul 2023 10:00:38 +0000 Subject: [PATCH 44/90] fix: missing imports --- tests/unit/api/models/repositories/test_repo_templates.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/api/models/repositories/test_repo_templates.py b/tests/unit/api/models/repositories/test_repo_templates.py index 84652ec0..90d26173 100644 --- a/tests/unit/api/models/repositories/test_repo_templates.py +++ b/tests/unit/api/models/repositories/test_repo_templates.py @@ -19,6 +19,7 @@ # SOFTWARE. import logging +import os import tempfile from typing import Dict, List @@ -26,6 +27,7 @@ from lifemonitor.api.models import repositories as repos from lifemonitor.api.models.repositories import templates +from lifemonitor.utils import boolean_value logger = logging.getLogger(__name__) From 9505580f316bc4a70b8e85d81bff464ecbf7b1d0 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 7 Jul 2023 10:02:29 +0000 Subject: [PATCH 45/90] fix: as a workaround, test repo templates separately --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index dddd13a7..a13197a5 100644 --- a/Makefile +++ b/Makefile @@ -263,13 +263,16 @@ start-aux-services: aux_images ro_crates docker-compose.extra.yml permissions ## run-tests: start-testing ## Run all tests in the Testing Environment @printf "\n$(bold)Running tests...$(reset)\n" ; \ USER_UID=$$(id -u) USER_GID=$$(id -g) \ - $(docker_compose) exec -T lmtests /bin/bash -c "pytest --durations=10 --color=yes tests" + $(docker_compose) exec -T lmtests /bin/bash -c "TEST_SKIP_REPO_TEMPLATES=true pytest --durations=10 --color=yes tests" \ + $(docker_compose) exec -T lmtests /bin/bash -c "pytest --durations=10 --color=yes tests/unit/api/models/repositories/test_repo_templates.py" tests: start-testing ## CI utility to setup, run tests and teardown a testing environment @printf "\n$(bold)Running tests...$(reset)\n" ; \ $(docker_compose) -f ./docker-compose.yml \ - exec -T lmtests /bin/bash -c "pytest --reruns 2 --reruns-delay 5 --durations=10 --color=yes tests --order-dependencies"; \ + exec -T lmtests /bin/bash -c "TEST_SKIP_REPO_TEMPLATES=true pytest --reruns 2 --reruns-delay 5 --durations=10 --color=yes --order-dependencies tests"; \ + $(docker_compose) -f ./docker-compose.yml \ + exec -T lmtests /bin/bash -c "pytest --reruns 2 --reruns-delay 5 --durations=10 --color=yes --order-dependencies tests/unit/api/models/repositories/test_repo_templates.py"; \ result=$$?; \ printf "\n$(bold)Teardown services...$(reset)\n" ; \ USER_UID=$$(id -u) USER_GID=$$(id -g) \ From 7a8b4d6f0623b622e12545dc3e40798b91bd513d Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 Jul 2023 12:47:19 +0000 Subject: [PATCH 46/90] fix: disable requests_cache --- .../api/models/repositories/templates/nextflow.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lifemonitor/api/models/repositories/templates/nextflow.py b/lifemonitor/api/models/repositories/templates/nextflow.py index a21a3877..1f28d58d 100644 --- a/lifemonitor/api/models/repositories/templates/nextflow.py +++ b/lifemonitor/api/models/repositories/templates/nextflow.py @@ -25,11 +25,20 @@ import re from typing import Dict, List, Optional +from nf_core.create import PipelineCreate + import lifemonitor.api.models.repositories as repos import lifemonitor.api.models.repositories.files as repo_files from . import WorkflowRepositoryTemplate +# disable requests_cache +try: + import requests_cache + requests_cache.patcher.uninstall_cache() +except ImportError: + pass + # set module level logger logger = logging.getLogger(__name__) @@ -69,8 +78,6 @@ def files(self) -> List[repo_files.base.RepositoryFile]: def generate(self, target_path: str = None) -> repos.LocalWorkflowRepository: target_path = target_path or self.local_path logger.debug("Rendering template files to %s...", target_path) - # name, description, author, version="1.0dev", no_git=False, force=False, outdir=None - from nf_core.create import PipelineCreate create_obj = PipelineCreate( re.sub(r"\s+", "", self.data.get("workflow_name")), self.data.get("workflow_description"), From cd1ea5b5eed321bec16a664cfe6f7fc2d61d6657 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 Jul 2023 15:22:50 +0000 Subject: [PATCH 47/90] Revert "fix: as a workaround, test repo templates separately" This reverts commit 73b1c9f32d3febf9297d8f78a97a970bb27d748b. --- Makefile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index a13197a5..dddd13a7 100644 --- a/Makefile +++ b/Makefile @@ -263,16 +263,13 @@ start-aux-services: aux_images ro_crates docker-compose.extra.yml permissions ## run-tests: start-testing ## Run all tests in the Testing Environment @printf "\n$(bold)Running tests...$(reset)\n" ; \ USER_UID=$$(id -u) USER_GID=$$(id -g) \ - $(docker_compose) exec -T lmtests /bin/bash -c "TEST_SKIP_REPO_TEMPLATES=true pytest --durations=10 --color=yes tests" \ - $(docker_compose) exec -T lmtests /bin/bash -c "pytest --durations=10 --color=yes tests/unit/api/models/repositories/test_repo_templates.py" + $(docker_compose) exec -T lmtests /bin/bash -c "pytest --durations=10 --color=yes tests" tests: start-testing ## CI utility to setup, run tests and teardown a testing environment @printf "\n$(bold)Running tests...$(reset)\n" ; \ $(docker_compose) -f ./docker-compose.yml \ - exec -T lmtests /bin/bash -c "TEST_SKIP_REPO_TEMPLATES=true pytest --reruns 2 --reruns-delay 5 --durations=10 --color=yes --order-dependencies tests"; \ - $(docker_compose) -f ./docker-compose.yml \ - exec -T lmtests /bin/bash -c "pytest --reruns 2 --reruns-delay 5 --durations=10 --color=yes --order-dependencies tests/unit/api/models/repositories/test_repo_templates.py"; \ + exec -T lmtests /bin/bash -c "pytest --reruns 2 --reruns-delay 5 --durations=10 --color=yes tests --order-dependencies"; \ result=$$?; \ printf "\n$(bold)Teardown services...$(reset)\n" ; \ USER_UID=$$(id -u) USER_GID=$$(id -g) \ From 0e0b373ccfd202fc7e0a3528800ad3b134cb1022 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 11 Jul 2023 09:35:22 +0000 Subject: [PATCH 48/90] fix: missing proxy header --- docker/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx.conf b/docker/nginx.conf index 1d808112..6bbe04dd 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -29,7 +29,7 @@ server { # resolver 127.0.0.11 ipv6=off valid=30s; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $http_host; + proxy_set_header Host lm:8000; # we don't want nginx trying to do something clever with # redirects, we set the Host: header above already. proxy_redirect off; From ea50d202e7e83927b1662d8ebfdfb5fee57faedb Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 11 Jul 2023 10:09:24 +0000 Subject: [PATCH 49/90] build(cfg): add redirect for the root path --- docker/nginx.conf | 5 +++++ docker/nginx.dev.conf | 5 +++++ k8s/templates/nginx-configmap.yaml | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/docker/nginx.conf b/docker/nginx.conf index 6bbe04dd..09379439 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -24,6 +24,11 @@ server { root /app/lifemonitor; } + # if the path matches to root, redirect to the account page + location = / { + return 301 https://$host:8443/account/; + } + # set proxy location location / { # resolver 127.0.0.11 ipv6=off valid=30s; diff --git a/docker/nginx.dev.conf b/docker/nginx.dev.conf index 3519c1f7..2036f017 100644 --- a/docker/nginx.dev.conf +++ b/docker/nginx.dev.conf @@ -37,6 +37,11 @@ server { proxy_set_header Host lm:8000; } + # if the path matches to root, redirect to the account page + location = / { + return 301 https://$host:8443/account/; + } + # set proxy location location / { # resolver 127.0.0.11 ipv6=off valid=30s; diff --git a/k8s/templates/nginx-configmap.yaml b/k8s/templates/nginx-configmap.yaml index eceb3e2e..2c01683f 100644 --- a/k8s/templates/nginx-configmap.yaml +++ b/k8s/templates/nginx-configmap.yaml @@ -33,6 +33,11 @@ data: location /static/ { root /app/lifemonitor; } + + # if the path matches to root, redirect to the account page + location = / { + return 301 https://$host:8443/account/; + } # set proxy location location / { From 9128680b0c6604a0f8a280267dc3052a31da7935 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 11 Jul 2023 13:26:25 +0000 Subject: [PATCH 50/90] fix(utils): allowto use insecure seek registry --- tests/wait-for-seek.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/wait-for-seek.sh b/tests/wait-for-seek.sh index c8eebda6..a33a4798 100755 --- a/tests/wait-for-seek.sh +++ b/tests/wait-for-seek.sh @@ -6,7 +6,7 @@ DEBUG=${DEBUG:-0} # Function to check if the server is available check_server() { - response=$(curl -Is "$url" | head -n 1) + response=$(curl -Is "$url" --insecure | head -n 1) status_code=$(echo "$response" | awk '{print $2}') if [ "$status_code" == "302" ]; then From dfb35f636305f44aab79247052d985c0454dd372 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 12 Jul 2023 08:04:19 +0000 Subject: [PATCH 51/90] feat(utils): allow extra domains in self-signed certs --- utils/certs/gencerts.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/utils/certs/gencerts.sh b/utils/certs/gencerts.sh index c38efcb3..57f9230f 100755 --- a/utils/certs/gencerts.sh +++ b/utils/certs/gencerts.sh @@ -22,9 +22,14 @@ if uname -a | grep -q Darwin; then fi IPADDRESSES="$(echo "${NETWORK_DATA}" | "${gsed}" -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p;' | "${gsed}" -e ':a;N;$!ba;s/\n/,/g')" -DOMAINS="lm,lm.local,lifemonitor,lifemonitor.local,lmtests,localhost,seek,nginx,wfhub" +DOMAINS="lm,lm.local,lifemonitor,lifemonitor.local,lmtests,seek,nginx,wfhub,$(hostname),localhost" IMAGE_NAME="crs4/minica" +# add extra domains +if [[ -n "${EXTRA_DOMAINS}" ]]; then + DOMAINS="${DOMAINS},${EXTRA_DOMAINS}" +fi + # script path current_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" From 1cdf6d62aa3d742eec59099bf54f56309fd74a67 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 12 Jul 2023 08:04:39 +0000 Subject: [PATCH 52/90] style: reformat --- utils/certs/gencerts.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/certs/gencerts.sh b/utils/certs/gencerts.sh index 57f9230f..977810a3 100755 --- a/utils/certs/gencerts.sh +++ b/utils/certs/gencerts.sh @@ -4,7 +4,7 @@ set -o errexit set -o nounset CA_NAME="ca" -if type -P ifconfig > /dev/null 2>&1; then +if type -P ifconfig >/dev/null 2>&1; then NETWORK_DATA="$(ifconfig)" else NETWORK_DATA="$(ip -oneline addr)" @@ -13,7 +13,7 @@ fi # check if GNU sed is available gsed=sed if uname -a | grep -q Darwin; then - if ! type -P gsed > /dev/null 2>&1; then + if ! type -P gsed >/dev/null 2>&1; then echo "GNU sed is not available. Please install it with 'brew install gnu-sed'" >&2 exit 1 else @@ -31,7 +31,7 @@ if [[ -n "${EXTRA_DOMAINS}" ]]; then fi # script path -current_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +current_path="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" # gen cmd cmd="minica -ca-cert \"${CA_NAME}.pem\" -ca-key \"${CA_NAME}.key\" -domains \"${DOMAINS}\" -ip-addresses \"${IPADDRESSES}\"" From 455a36bdabfa961c8b1ebb5459bd8a0a6090175c Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 12 Jul 2023 08:10:15 +0000 Subject: [PATCH 53/90] fix: unbound variable --- utils/certs/gencerts.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/certs/gencerts.sh b/utils/certs/gencerts.sh index 977810a3..42dd56b0 100755 --- a/utils/certs/gencerts.sh +++ b/utils/certs/gencerts.sh @@ -26,6 +26,7 @@ DOMAINS="lm,lm.local,lifemonitor,lifemonitor.local,lmtests,seek,nginx,wfhub,$(ho IMAGE_NAME="crs4/minica" # add extra domains +EXTRA_DOMAINS="${EXTRA_DOMAINS:-}" if [[ -n "${EXTRA_DOMAINS}" ]]; then DOMAINS="${DOMAINS},${EXTRA_DOMAINS}" fi From 5a6a435e05cd75b724a961dd727d40e8e2c0dd4a Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 12 Jul 2023 09:40:02 +0000 Subject: [PATCH 54/90] style: reformat --- docker/nginx.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/nginx.conf b/docker/nginx.conf index 09379439..5a14c3f2 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -12,12 +12,12 @@ server { #server_name localhost; keepalive_timeout 60; etag on; - + ssl_certificate /nginx/certs/lm.crt; ssl_certificate_key /nginx/certs/lm.key; # force HTTP traffic to HTTPS - error_page 497 https://$host:8443$request_uri; + error_page 497 https://$host:8443$request_uri; # set static files location location /static/ { From 5bf310dd65ef6489e25e10ce712238491ce7bf20 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 13 Jul 2023 14:01:41 +0000 Subject: [PATCH 55/90] build(k8s): automatically generate TLS secret if it doesn't exists --- k8s/templates/_helpers.tpl | 22 +++++++++++++++++++++- k8s/templates/tls.yaml | 10 ++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 k8s/templates/tls.yaml diff --git a/k8s/templates/_helpers.tpl b/k8s/templates/_helpers.tpl index da9ab153..143607a5 100644 --- a/k8s/templates/_helpers.tpl +++ b/k8s/templates/_helpers.tpl @@ -61,6 +61,14 @@ Define lifemonitor image {{- end }} +{{/* +Define lifemonitor TLS secret name +*/}} +{{- define "chart.lifemonitor.tls" -}} +{{- printf "%s-tls" .Release.Name }} +{{- end }} + + {{/* Create the name of the service account to use */}} @@ -112,7 +120,7 @@ Define volumes shared by some pods. {{- define "lifemonitor.common-volume" -}} - name: lifemonitor-tls secret: - secretName: lifemonitor-tls + secretName: {{ include "chart.lifemonitor.tls" . }} - name: lifemonitor-settings secret: secretName: {{ include "chart.fullname" . }}-settings @@ -153,3 +161,15 @@ Define mount points shared by some pods. {{- end -}} {{- end -}} {{- end -}} + + +{{/* +Generate certificates for the LifeMonitor Api Server . +*/}} +{{- define "gen-certs" -}} +{{- $altNames := list ( printf "%s.%s" (include "chart.name" .) .Release.Namespace ) ( printf "%s.%s.svc" (include "chart.name" .) .Release.Namespace ) -}} +{{- $ca := genCA "lifemonitor-ca" 365 -}} +{{- $cert := genSignedCert ( include "chart.name" . ) nil $altNames 365 $ca -}} +tls.crt: {{ $cert.Cert | b64enc }} +tls.key: {{ $cert.Key | b64enc }} +{{- end -}} \ No newline at end of file diff --git a/k8s/templates/tls.yaml b/k8s/templates/tls.yaml new file mode 100644 index 00000000..72566daf --- /dev/null +++ b/k8s/templates/tls.yaml @@ -0,0 +1,10 @@ +{{- $existingTLS := (lookup "v1" "Secret" .Release.Namespace ( include "chart.lifemonitor.tls" . )) }} +{{- if not $existingTLS }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "chart.lifemonitor.tls" . }} +type: kubernetes.io/tls +data: +{{ ( include "gen-certs" . ) | indent 2 }} +{{- end -}} \ No newline at end of file From 6ebbb0a315c73cfa32466d5bcb60a52ff218167d Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 13 Jul 2023 14:03:14 +0000 Subject: [PATCH 56/90] build(k8s): always delete existing init JOB before release creation --- k8s/templates/job-init.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/templates/job-init.yaml b/k8s/templates/job-init.yaml index 053a2d70..83b50aff 100644 --- a/k8s/templates/job-init.yaml +++ b/k8s/templates/job-init.yaml @@ -11,7 +11,7 @@ metadata: # job is considered part of the release. "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-weight": "-5" - "helm.sh/hook-delete-policy": hook-succeeded + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: template: spec: From 957587447bb0a22a263465569c68fbc6fdbc88ab Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 Jul 2023 10:27:23 +0000 Subject: [PATCH 57/90] build(k8s): update default image --- k8s/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/values.yaml b/k8s/values.yaml index cf2fd19f..a540aec1 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -129,7 +129,7 @@ backup: lifemonitor: replicaCount: 1 - image: &lifemonitorImage crs4/lifemonitor:0.4.0 + image: &lifemonitorImage crs4/lifemonitor:0.11.4 imagePullPolicy: &lifemonitorImagePullPolicy Always imagePullSecrets: [] From 7a967d2745d01100ceff240922978f30879887aa Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 Jul 2023 10:27:51 +0000 Subject: [PATCH 58/90] build(k8s): decrease default storage size --- k8s/values.yaml | 417 +----------------------------------------------- 1 file changed, 4 insertions(+), 413 deletions(-) diff --git a/k8s/values.yaml b/k8s/values.yaml index a540aec1..79049228 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -167,7 +167,7 @@ lifemonitor: persistence: storageClass: *storageClass - storageSize: 8Gi + storageSize: 1Gi # Enable/Disable the pod to test connection to the LifeMonitor back-end enableTestConnection: false @@ -430,7 +430,7 @@ postgresql: storageClass: *storageClass accessModes: - ReadWriteOnce - size: 8Gi + size: 1Gi annotations: {} ## selector can be used to match an existing PersistentVolume ## selector: @@ -438,415 +438,6 @@ postgresql: ## app: my-app selector: {} -nginx: - ## Global Docker image parameters - ## Please, note that this will override the image parameters, including dependencies, configured to use the global value - ## Current available global Docker image parameters: imageRegistry and imagePullSecrets - ## - # global: - # imageRegistry: myRegistryName - # imagePullSecrets: - # - myRegistryKeySecretName - - ## Bitnami NGINX image version - ## ref: https://hub.docker.com/r/bitnami/nginx/tags/ - ## - image: - registry: docker.io - repository: bitnami/nginx - tag: 1.19.5-debian-10-r0 - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ - ## E.g.: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - ## Set to true if you would like to see extra information on logs - ## - debug: false - - ## String to partially override nginx.fullname template (will maintain the release name) - ## - nameOverride: "nginx" - - ## String to fully override nginx.fullname template - ## - # fullnameOverride: "" - - ## Kubernetes Cluster Domain - ## - clusterDomain: cluster.local - - ## Extra objects to deploy (value evaluated as a template) - ## - extraDeploy: [] - - ## Add labels to all the deployed resources - ## - commonLabels: {} - - ## Add annotations to all the deployed resources - ## - commonAnnotations: {} - - ## Command and args for running the container (set to default if not set). Use array form - ## - # command: - # args: - - ## Additional environment variables to set - ## E.g: - ## extraEnvVars: - ## - name: FOO - ## value: BAR - ## - extraEnvVars: [] - - ## ConfigMap with extra environment variables - ## - # extraEnvVarsCM: - - ## Secret with extra environment variables - ## - # extraEnvVarsSecret: - - ## Custom server block to be added to NGINX configuration - # serverBlock: |- - # # set upstream server - # upstream lm_app { - # # fail_timeout=0 means we always retry an upstream even if it failed - # # to return a good HTTP response - # server lifemonitor-backend:8000 fail_timeout=0; - # } - - # server { - # listen 0.0.0.0:8080 default_server; - # client_max_body_size 4G; - # # set the correct host(s) for your site - # server_name localhost; - # keepalive_timeout 60; - - # #ssl_certificate /nginx/certs/lm.crt; - # #ssl_certificate_key /nginx/certs/lm.key; - - # # force HTTP traffic to HTTPS - # error_page 497 https://$host:8443$request_uri; - - # # set proxy location - # location / { - # #resolver 127.0.0.11 ipv6=off valid=30s; - # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - # proxy_set_header X-Forwarded-Proto $scheme; - # proxy_set_header Host $http_host; - # # we don't want nginx trying to do something clever with - # # redirects, we set the Host: header above already. - # proxy_redirect off; - # proxy_pass https://lm_app; - # } - # } - - ## ConfigMap with custom server block to be added to NGINX configuration - ## NOTE: This will override serverBlock - ## - existingServerBlockConfigmap: "lifemonitor-nginx-configmap" - - ## Name of existing ConfigMap with the server static site content - ## - # staticSiteConfigmap - - ## Name of existing PVC with the server static site content - ## NOTE: This will override staticSiteConfigmap - ## - #staticSitePVC: - - # Set init containers - initContainers: - - name: init-static-files - image: *lifemonitorImage - imagePullPolicy: *lifemonitorImagePullPolicy - command: ["/bin/bash", "-c"] - args: - - | - mkdir -p /app/lifemonitor - cp -a /lm/lifemonitor/static /app/lifemonitor/static - cp -a /lm/specs /app/specs - volumeMounts: - - mountPath: /app - name: static-files - - ## Number of replicas to deploy - ## - replicaCount: 1 - - ## Pod extra labels - ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ - ## - podLabels: {} - - ## Pod annotations - ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ - ## - podAnnotations: {} - - ## Pod affinity preset - ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity - ## Allowed values: soft, hard - ## - podAffinityPreset: "" - - ## Pod anti-affinity preset - ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity - ## Allowed values: soft, hard - ## - podAntiAffinityPreset: soft - - ## Node affinity preset - ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity - ## Allowed values: soft, hard - ## - nodeAffinityPreset: - ## Node affinity type - ## Allowed values: soft, hard - type: "" - ## Node label key to match - ## E.g. - ## key: "kubernetes.io/e2e-az-name" - ## - key: "" - ## Node label values to match - ## E.g. - ## values: - ## - e2e-az1 - ## - e2e-az2 - ## - values: [] - - ## Affinity for pod assignment - ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity - ## Note: podAffinityPreset, podAntiAffinityPreset, and nodeAffinityPreset will be ignored when it's set - ## - affinity: {} - - ## Node labels for pod assignment. Evaluated as a template. - ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ - ## - nodeSelector: {} - - ## Tolerations for pod assignment. Evaluated as a template. - ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ - ## - tolerations: {} - - ## NGINX pods' Security Context. - ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod - ## - podSecurityContext: - enabled: false - runAsUser: 1001 - runAsNonRoot: true - ## sysctl settings - ## Example: - ## sysctls: - ## - name: net.core.somaxconn - ## value: "10000" - ## - sysctls: {} - - ## NGINX Core containers' Security Context (only main container). - ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container - ## - containerSecurityContext: - enabled: false - fsGroup: 1001 - - ## Configures the ports NGINX listens on - ## - containerPorts: - http: 8080 - - ## NGINX containers' resource requests and limits - ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ - ## - resources: - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - limits: {} - # cpu: 100m - # memory: 128Mi - requests: {} - # cpu: 100m - # memory: 128Mi - - ## NGINX containers' liveness and readiness probes. - ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes - ## - livenessProbe: - enabled: true - initialDelaySeconds: 30 - timeoutSeconds: 5 - periodSeconds: 10 - failureThreshold: 6 - successThreshold: 1 - readinessProbe: - enabled: true - initialDelaySeconds: 5 - timeoutSeconds: 3 - periodSeconds: 5 - failureThreshold: 3 - successThreshold: 1 - - ## Custom Liveness probe - ## - customLivenessProbe: {} - - ## Custom Rediness probe - ## - customReadinessProbe: {} - - ## Autoscaling parameters - ## - autoscaling: - enabled: false - # minReplicas: 1 - # maxReplicas: 10 - # targetCPU: 50 - # targetMemory: 50 - - ## Array to add extra volumes (evaluated as a template) - ## - extraVolumes: - - name: static-files - emptyDir: {} - - ## Array to add extra mounts (normally used with extraVolumes, evaluated as a template) - ## - extraVolumeMounts: - - mountPath: /app - name: static-files - - ## NGINX Service properties - ## - service: - ## Service type - ## - type: ClusterIP - - ## HTTP Port - ## - port: 8080 - - ## Set the LoadBalancer service type to internal only. - ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer - ## - # loadBalancerIP: - - ## Provide any additional annotations which may be required. This can be used to - ## set the LoadBalancer service type to internal only. - ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer - ## - annotations: {} - - ## Enable client source IP preservation - ## ref http://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip - ## - externalTrafficPolicy: Local - - ## Prometheus Exporter / Metrics - ## - metrics: - enabled: false - - ## Bitnami NGINX Prometheus Exporter image - ## ref: https://hub.docker.com/r/bitnami/nginx-exporter/tags/ - ## - image: - registry: docker.io - repository: bitnami/nginx-exporter - tag: 0.8.0-debian-10-r147 - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ - ## - # pullSecrets: - # - myRegistryKeySecretName - - ## Prometheus exporter pods' annotation and labels - ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ - ## - podAnnotations: {} - - ## Prometheus exporter service parameters - ## - service: - ## NGINX Prometheus exporter port - ## - port: 9113 - ## Annotations for the Prometheus exporter service - ## - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "{{ .Values.metrics.service.port }}" - - ## NGINX Prometheus exporter resource requests and limits - ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ - ## - resources: - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - limits: {} - # cpu: 100m - # memory: 128Mi - requests: {} - # cpu: 100m - # memory: 128Mi - - ## Prometheus Operator ServiceMonitor configuration - ## - serviceMonitor: - enabled: false - ## Namespace in which Prometheus is running - ## - # namespace: monitoring - - ## Interval at which metrics should be scraped. - ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint - ## - # interval: 10s - - ## Timeout after which the scrape is ended - ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint - ## - # scrapeTimeout: 10s - - ## ServiceMonitor selector labels - ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration - ## - # selector: - # prometheus: my-prometheus - - ## Pod Disruption Budget configuration - ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ - ## - pdb: - create: false - ## Min number of pods that must still be available after the eviction - ## - minAvailable: 1 - ## Max number of pods that can be unavailable after the eviction - ## - # maxUnavailable: 1 redis: image: @@ -1157,7 +748,7 @@ redis: - ReadWriteOnce ## @param master.persistence.size Persistent Volume size ## - size: 8Gi + size: 4Gi ## @param master.persistence.annotations Additional custom annotations for the PVC ## annotations: {} @@ -1466,7 +1057,7 @@ redis: - ReadWriteOnce ## @param replica.persistence.size Persistent Volume size ## - size: 8Gi + size: 4Gi ## @param replica.persistence.annotations Additional custom annotations for the PVC ## annotations: {} From 0a2c9ed63ea52099e89f8baf9b1ea7036fbb13cc Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 Jul 2023 13:57:23 +0000 Subject: [PATCH 59/90] build(k8s): config default storage classes --- k8s/values.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/k8s/values.yaml b/k8s/values.yaml index 79049228..89434f1f 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -14,7 +14,9 @@ externalServerName: &hostname api.lifemonitor.eu webappUrl: &webapp_url https://app.lifemonitor.eu # global storage class -storageClass: &storageClass "-" +storageClass: + default: &defaultStorageClass "-" + readWriteManyStorageClass: &readWriteManyStorageClass "-" remoteStorage: enabled: false @@ -166,7 +168,7 @@ lifemonitor: port: 8000 persistence: - storageClass: *storageClass + storageClass: *defaultStorageClass storageSize: 1Gi # Enable/Disable the pod to test connection to the LifeMonitor back-end @@ -427,7 +429,7 @@ postgresql: ## subPath: "" - storageClass: *storageClass + storageClass: *defaultStorageClass accessModes: - ReadWriteOnce size: 1Gi @@ -741,7 +743,7 @@ redis: ## If set to "-", storageClassName: "", which disables dynamic provisioning ## If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner ## - storageClass: "" + storageClass: *defaultStorageClass ## @param master.persistence.accessModes [array] Persistent Volume access modes ## accessModes: @@ -1050,7 +1052,7 @@ redis: ## If set to "-", storageClassName: "", which disables dynamic provisioning ## If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner ## - storageClass: "" + storageClass: *defaultStorageClass ## @param replica.persistence.accessModes [array] Persistent Volume access modes ## accessModes: From cc38e87680a7abf5801d715404100d41a73dfd65 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 Jul 2023 14:00:02 +0000 Subject: [PATCH 60/90] build(k8s): auto configuration of PVCs --- k8s/templates/_helpers.tpl | 21 +++++++++++++++++++++ k8s/templates/pvc-backup-data.yaml | 14 ++++++++++++++ k8s/templates/pvc-logs-data.yaml | 14 ++++++++++++++ k8s/templates/pvc-workflows-data.yaml | 14 ++++++++++++++ 4 files changed, 63 insertions(+) create mode 100644 k8s/templates/pvc-backup-data.yaml create mode 100644 k8s/templates/pvc-logs-data.yaml create mode 100644 k8s/templates/pvc-workflows-data.yaml diff --git a/k8s/templates/_helpers.tpl b/k8s/templates/_helpers.tpl index 143607a5..78c32cf8 100644 --- a/k8s/templates/_helpers.tpl +++ b/k8s/templates/_helpers.tpl @@ -69,6 +69,27 @@ Define lifemonitor TLS secret name {{- end }} +{{/* +Define volume name of LifeMonitor backup data +*/}} +{{- define "chart.lifemonitor.data.backup" -}} +{{- printf "data-%s-backup" .Release.Name }} +{{- end }} + +{{/* +Define volume name of LifeMonitor workflows data +*/}} +{{- define "chart.lifemonitor.data.workflows" -}} +{{- printf "data-%s-workflows" .Release.Name }} +{{- end }} + +{{/* +Define volume name of LifeMonitor logs data +*/}} +{{- define "chart.lifemonitor.data.logs" -}} +{{- printf "data-%s-logs" .Release.Name }} +{{- end }} + {{/* Create the name of the service account to use */}} diff --git a/k8s/templates/pvc-backup-data.yaml b/k8s/templates/pvc-backup-data.yaml new file mode 100644 index 00000000..7f8f86bb --- /dev/null +++ b/k8s/templates/pvc-backup-data.yaml @@ -0,0 +1,14 @@ +{{- $dataBackupExists := (lookup "v1" "PersistentVolumeClaim" .Release.Namespace ( include "chart.lifemonitor.data.backup" . ) ) }} +{{- if not $dataBackupExists }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "chart.lifemonitor.data.backup" . }} +spec: + storageClassName: {{ .Values.storageClass.readWriteManyStorageClass }} + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi +{{- end -}} \ No newline at end of file diff --git a/k8s/templates/pvc-logs-data.yaml b/k8s/templates/pvc-logs-data.yaml new file mode 100644 index 00000000..55dff936 --- /dev/null +++ b/k8s/templates/pvc-logs-data.yaml @@ -0,0 +1,14 @@ +{{- $dataLogsExists := (lookup "v1" "PersistentVolumeClaim" .Release.Namespace ( include "chart.lifemonitor.data.logs" . ) ) }} +{{- if not $dataLogsExists }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: data-api-logs +spec: + storageClassName: nfs-client + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi +{{- end -}} diff --git a/k8s/templates/pvc-workflows-data.yaml b/k8s/templates/pvc-workflows-data.yaml new file mode 100644 index 00000000..effaa03e --- /dev/null +++ b/k8s/templates/pvc-workflows-data.yaml @@ -0,0 +1,14 @@ +{{- $dataWorkflowsExists := (lookup "v1" "PersistentVolumeClaim" .Release.Namespace ( include "chart.lifemonitor.data.workflows" . ) ) }} +{{- if not $dataWorkflowsExists }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "chart.lifemonitor.data.workflows" . }} +spec: + storageClassName: {{ .Values.storageClass.readWriteManyStorageClass }} + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi +{{- end -}} \ No newline at end of file From 383ca9862a03e91940d3bb9a2a165409d4e0b820 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 19 Jul 2023 14:09:01 +0000 Subject: [PATCH 61/90] fix: wrong property to define storageClass name --- k8s/templates/pvc-backup-data.yaml | 2 +- k8s/templates/pvc-logs-data.yaml | 2 +- k8s/templates/pvc-workflows-data.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/templates/pvc-backup-data.yaml b/k8s/templates/pvc-backup-data.yaml index 7f8f86bb..1f3994f6 100644 --- a/k8s/templates/pvc-backup-data.yaml +++ b/k8s/templates/pvc-backup-data.yaml @@ -5,7 +5,7 @@ kind: PersistentVolumeClaim metadata: name: {{ include "chart.lifemonitor.data.backup" . }} spec: - storageClassName: {{ .Values.storageClass.readWriteManyStorageClass }} + storageClassName: {{ .Values.readWriteManyStorageClass }} accessModes: - ReadWriteMany resources: diff --git a/k8s/templates/pvc-logs-data.yaml b/k8s/templates/pvc-logs-data.yaml index 55dff936..13ca0616 100644 --- a/k8s/templates/pvc-logs-data.yaml +++ b/k8s/templates/pvc-logs-data.yaml @@ -5,7 +5,7 @@ kind: PersistentVolumeClaim metadata: name: data-api-logs spec: - storageClassName: nfs-client + storageClassName: {{ .Values.readWriteManyStorageClass }} accessModes: - ReadWriteMany resources: diff --git a/k8s/templates/pvc-workflows-data.yaml b/k8s/templates/pvc-workflows-data.yaml index effaa03e..710901b9 100644 --- a/k8s/templates/pvc-workflows-data.yaml +++ b/k8s/templates/pvc-workflows-data.yaml @@ -5,7 +5,7 @@ kind: PersistentVolumeClaim metadata: name: {{ include "chart.lifemonitor.data.workflows" . }} spec: - storageClassName: {{ .Values.storageClass.readWriteManyStorageClass }} + storageClassName: {{ .Values.readWriteManyStorageClass }} accessModes: - ReadWriteMany resources: From 5311e0964a2c2ac992cbcff0b13cc6494b199150 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 19 Jul 2023 14:10:58 +0000 Subject: [PATCH 62/90] fix: dynamic PVC name --- k8s/templates/pvc-logs-data.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/templates/pvc-logs-data.yaml b/k8s/templates/pvc-logs-data.yaml index 13ca0616..098dc472 100644 --- a/k8s/templates/pvc-logs-data.yaml +++ b/k8s/templates/pvc-logs-data.yaml @@ -3,7 +3,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: data-api-logs + name: {{ include "chart.lifemonitor.data.logs" . }} spec: storageClassName: {{ .Values.readWriteManyStorageClass }} accessModes: From a020ef1e82dad13ee9352196454c34c4b4befc07 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 20 Jul 2023 14:36:46 +0000 Subject: [PATCH 63/90] build(k8s): reconfigure storage classes --- k8s/values.yaml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/k8s/values.yaml b/k8s/values.yaml index 89434f1f..e651da5f 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -13,11 +13,13 @@ externalServerName: &hostname api.lifemonitor.eu # Base URL of the LifeMonitor web app associated with this back-end instance webappUrl: &webapp_url https://app.lifemonitor.eu -# global storage class -storageClass: - default: &defaultStorageClass "-" +# global storage classes +global: + readWriteOnceStorageClass: &readWriteOnceStorageClass "-" readWriteManyStorageClass: &readWriteManyStorageClass "-" + storageClass: *readWriteOnceStorageClass +# remote storage settings remoteStorage: enabled: false s3_endpoint_url: "https://your-s3-storage-url" @@ -168,7 +170,7 @@ lifemonitor: port: 8000 persistence: - storageClass: *defaultStorageClass + storageClass: *readWriteOnceStorageClass storageSize: 1Gi # Enable/Disable the pod to test connection to the LifeMonitor back-end @@ -429,7 +431,7 @@ postgresql: ## subPath: "" - storageClass: *defaultStorageClass + storageClass: *readWriteOnceStorageClass accessModes: - ReadWriteOnce size: 1Gi @@ -440,7 +442,6 @@ postgresql: ## app: my-app selector: {} - redis: image: registry: docker.io @@ -743,7 +744,7 @@ redis: ## If set to "-", storageClassName: "", which disables dynamic provisioning ## If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner ## - storageClass: *defaultStorageClass + storageClass: *readWriteOnceStorageClass ## @param master.persistence.accessModes [array] Persistent Volume access modes ## accessModes: @@ -1052,7 +1053,7 @@ redis: ## If set to "-", storageClassName: "", which disables dynamic provisioning ## If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner ## - storageClass: *defaultStorageClass + storageClass: *readWriteOnceStorageClass ## @param replica.persistence.accessModes [array] Persistent Volume access modes ## accessModes: From 39419f5d392d9cad1b9894ae14be6a12ab1a9c1d Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 20 Jul 2023 14:43:57 +0000 Subject: [PATCH 64/90] refactor(k8s): rename deployemnt files --- .../{backend-deployment.yaml => backend.deployment.yaml} | 9 ++++++++- .../{worker-deployment.yaml => worker.deployment.yaml} | 8 +++++++- .../{wss-deployment.yaml => wss.deployment.yaml} | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) rename k8s/templates/{backend-deployment.yaml => backend.deployment.yaml} (91%) rename k8s/templates/{worker-deployment.yaml => worker.deployment.yaml} (93%) rename k8s/templates/{wss-deployment.yaml => wss.deployment.yaml} (98%) diff --git a/k8s/templates/backend-deployment.yaml b/k8s/templates/backend.deployment.yaml similarity index 91% rename from k8s/templates/backend-deployment.yaml rename to k8s/templates/backend.deployment.yaml index 9d621d9f..041f6a89 100644 --- a/k8s/templates/backend-deployment.yaml +++ b/k8s/templates/backend.deployment.yaml @@ -16,13 +16,17 @@ spec: template: metadata: annotations: - checksum/settings: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + checksum/settings: {{ include (print $.Template.BasePath "/settings.secret.yaml") . | sha256sum }} {{- with .Values.lifemonitor.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} labels: {{- include "chart.selectorLabels" . | nindent 8 }} app.kubernetes.io/component: backend + # prometheus.io/scrape: 'true' + # prometheus.io/path: 'metrics' + # prometheus.io/port: '9090' + # prometheus.io/scheme: 'http' spec: {{- with .Values.lifemonitor.imagePullSecrets }} imagePullSecrets: @@ -57,6 +61,9 @@ spec: - name: http containerPort: 8000 protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP livenessProbe: httpGet: scheme: HTTPS diff --git a/k8s/templates/worker-deployment.yaml b/k8s/templates/worker.deployment.yaml similarity index 93% rename from k8s/templates/worker-deployment.yaml rename to k8s/templates/worker.deployment.yaml index 446fa029..67a4acc6 100644 --- a/k8s/templates/worker-deployment.yaml +++ b/k8s/templates/worker.deployment.yaml @@ -18,12 +18,16 @@ spec: template: metadata: annotations: - checksum/settings: {{ include (print $.Template.BasePath "/secret.yaml") $ | sha256sum }} + checksum/settings: {{ include (print $.Template.BasePath "/settings.secret.yaml") $ | sha256sum }} {{- with $.Values.worker.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} labels: {{- include "chart.selectorLabels" $ | nindent 8 }} + prometheus.io/scrape: 'true' + prometheus.io/path: 'metrics' + prometheus.io/port: '9191' + prometheus.io/scheme: 'http' spec: {{- with $.Values.worker.imagePullSecrets }} imagePullSecrets: @@ -64,6 +68,8 @@ spec: {{ else }} value: {{ $queue.name }} {{ end }} + ports: + - containerPort: 9191 volumeMounts: {{- include "lifemonitor.common-volume-mounts" $ | nindent 12 }} # livenessProbe: diff --git a/k8s/templates/wss-deployment.yaml b/k8s/templates/wss.deployment.yaml similarity index 98% rename from k8s/templates/wss-deployment.yaml rename to k8s/templates/wss.deployment.yaml index 82bb8869..074a41f5 100644 --- a/k8s/templates/wss-deployment.yaml +++ b/k8s/templates/wss.deployment.yaml @@ -16,7 +16,7 @@ spec: template: metadata: annotations: - checksum/settings: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + checksum/settings: {{ include (print $.Template.BasePath "/settings.secret.yaml") . | sha256sum }} {{- with .Values.lifemonitor.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} From b411a7170de4772a050b9c8d482677d2d4e4e153 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 20 Jul 2023 14:45:18 +0000 Subject: [PATCH 65/90] refactor(k8s): rename job files --- k8s/templates/{job-backup.yaml => backup.job.yaml} | 0 k8s/templates/{job-init.yaml => init.job.yaml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename k8s/templates/{job-backup.yaml => backup.job.yaml} (100%) rename k8s/templates/{job-init.yaml => init.job.yaml} (100%) diff --git a/k8s/templates/job-backup.yaml b/k8s/templates/backup.job.yaml similarity index 100% rename from k8s/templates/job-backup.yaml rename to k8s/templates/backup.job.yaml diff --git a/k8s/templates/job-init.yaml b/k8s/templates/init.job.yaml similarity index 100% rename from k8s/templates/job-init.yaml rename to k8s/templates/init.job.yaml From 745d410cd82b965db659304f067c8e6832c0b7d7 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 20 Jul 2023 14:47:08 +0000 Subject: [PATCH 66/90] refactor(k8s): rename pvc files --- k8s/pvc-backend-backup.yaml | 10 ---------- k8s/pvc-backend-data.yaml | 10 ---------- .../{pvc-backup-data.yaml => data-backup.pvc.yaml} | 0 .../{pvc-logs-data.yaml => data-logs.pvc.yaml} | 0 ...pvc-workflows-data.yaml => data-workflows.pvc.yaml} | 0 5 files changed, 20 deletions(-) delete mode 100644 k8s/pvc-backend-backup.yaml delete mode 100644 k8s/pvc-backend-data.yaml rename k8s/templates/{pvc-backup-data.yaml => data-backup.pvc.yaml} (100%) rename k8s/templates/{pvc-logs-data.yaml => data-logs.pvc.yaml} (100%) rename k8s/templates/{pvc-workflows-data.yaml => data-workflows.pvc.yaml} (100%) diff --git a/k8s/pvc-backend-backup.yaml b/k8s/pvc-backend-backup.yaml deleted file mode 100644 index 2dc8a2b8..00000000 --- a/k8s/pvc-backend-backup.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: data-api-backup -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 1Gi diff --git a/k8s/pvc-backend-data.yaml b/k8s/pvc-backend-data.yaml deleted file mode 100644 index 17db9368..00000000 --- a/k8s/pvc-backend-data.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: data-api-workflows -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 1Gi diff --git a/k8s/templates/pvc-backup-data.yaml b/k8s/templates/data-backup.pvc.yaml similarity index 100% rename from k8s/templates/pvc-backup-data.yaml rename to k8s/templates/data-backup.pvc.yaml diff --git a/k8s/templates/pvc-logs-data.yaml b/k8s/templates/data-logs.pvc.yaml similarity index 100% rename from k8s/templates/pvc-logs-data.yaml rename to k8s/templates/data-logs.pvc.yaml diff --git a/k8s/templates/pvc-workflows-data.yaml b/k8s/templates/data-workflows.pvc.yaml similarity index 100% rename from k8s/templates/pvc-workflows-data.yaml rename to k8s/templates/data-workflows.pvc.yaml From 8f34c6d414f31b93042e334ba1dba4e6d3cde2df Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 20 Jul 2023 14:47:39 +0000 Subject: [PATCH 67/90] refactor(k8s): rename secret files --- k8s/templates/{secret.yaml => settings.secret.yaml} | 0 k8s/templates/{tls.yaml => tls.secret.yaml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename k8s/templates/{secret.yaml => settings.secret.yaml} (100%) rename k8s/templates/{tls.yaml => tls.secret.yaml} (100%) diff --git a/k8s/templates/secret.yaml b/k8s/templates/settings.secret.yaml similarity index 100% rename from k8s/templates/secret.yaml rename to k8s/templates/settings.secret.yaml diff --git a/k8s/templates/tls.yaml b/k8s/templates/tls.secret.yaml similarity index 100% rename from k8s/templates/tls.yaml rename to k8s/templates/tls.secret.yaml From 904a91dd35beb5afc57897d1f66e72944a66414f Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 20 Jul 2023 16:43:00 +0000 Subject: [PATCH 68/90] build(k8s): replace deprecated ingress annotation --- k8s/values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/k8s/values.yaml b/k8s/values.yaml index e651da5f..0ebd7cdf 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -40,7 +40,8 @@ serviceAccount: ingress: enabled: false annotations: - kubernetes.io/ingress.class: nginx + # kubernetes.io/ingress.class: nginx + spec.ingressClassName: nginx #nginx.ingress.kubernetes.io/rewrite-target: / # kubernetes.io/tls-acme: "true" hosts: From fbf2de85f6749535110be1e05bd0cb3ec4187cb8 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 20 Jul 2023 18:18:19 +0000 Subject: [PATCH 69/90] fix(k8s): adopt new way of setting ingress class --- k8s/templates/ingress.yaml | 11 +++++++++++ k8s/values.yaml | 5 ++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/k8s/templates/ingress.yaml b/k8s/templates/ingress.yaml index 9b6b32bc..724178a3 100644 --- a/k8s/templates/ingress.yaml +++ b/k8s/templates/ingress.yaml @@ -14,6 +14,17 @@ metadata: {{- with .Values.ingress.annotations }} annotations: {{- toYaml . | nindent 4 }} + # new way of setting the ingress class + {{- if semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion -}} + spec.ingressClassName: {{ .Values.ingress.className }} + {{- else -}} + # old way of setting the ingress class + kubernetes.io/ingress.class: {{ .Values.ingress.className }} + {{- end -}} + nginx.ingress.kubernetes.io/affinity: "cookie" + nginx.ingress.kubernetes.io/session-cookie-name: "lm-api-back-end" + nginx.ingress.kubernetes.io/session-cookie-expires: "172800" + nginx.ingress.kubernetes.io/session-cookie-max-age: "172800" {{- end }} spec: {{- if .Values.ingress.tls }} diff --git a/k8s/values.yaml b/k8s/values.yaml index 0ebd7cdf..0039c075 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -39,10 +39,9 @@ serviceAccount: # from outside the cluster ingress: enabled: false + className: nginx annotations: - # kubernetes.io/ingress.class: nginx - spec.ingressClassName: nginx - #nginx.ingress.kubernetes.io/rewrite-target: / + # nginx.ingress.kubernetes.io/rewrite-target: / # kubernetes.io/tls-acme: "true" hosts: - host: *hostname From ca4b272ad2ccf410c8f5daa6327791c111d8aebf Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 24 Jul 2023 15:12:43 +0000 Subject: [PATCH 70/90] refactor: update ingress definition --- k8s/templates/ingress.yaml | 80 ++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 34 deletions(-) diff --git a/k8s/templates/ingress.yaml b/k8s/templates/ingress.yaml index 724178a3..704f5e3c 100644 --- a/k8s/templates/ingress.yaml +++ b/k8s/templates/ingress.yaml @@ -1,55 +1,67 @@ {{- if .Values.ingress.enabled -}} -{{- $fullName := include "chart.fullname" . -}} -{{- $svcPort := .Values.lifemonitor.service.port -}} -{{- if semverCompare ">=1.17-0" .Capabilities.KubeVersion.GitVersion -}} +{{- $fullname := include "chart.fullname" . -}} +{{- $serviceNewStyle := semverCompare ">=1.18.0" .Capabilities.KubeVersion.GitVersion -}} +{{- if semverCompare ">=1.17-0" .Capabilities.KubeVersion.GitVersion }} apiVersion: networking.k8s.io/v1 -{{- else -}} +{{- else }} apiVersion: extensions/v1beta1 {{- end }} kind: Ingress metadata: - name: {{ $fullName }} + name: {{ $fullname }} labels: - {{- include "chart.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} + app.kubernetes.io/name: {{ include "chart.name" . }} + helm.sh/chart: {{ include "chart.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} annotations: + {{- with .Values.ingress.annotations }} {{- toYaml . | nindent 4 }} - # new way of setting the ingress class - {{- if semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion -}} - spec.ingressClassName: {{ .Values.ingress.className }} - {{- else -}} + {{- end }} + # nginx.ingress.kubernetes.io/rewrite-target: / + # nginx.ingress.kubernetes.io/affinity: "cookie" + # traefik.ingress.kubernetes.io/service.sticky.cookie: "true" + # nginx.ingress.kubernetes.io/session-cookie-name: "lm-api-back-end" + # nginx.ingress.kubernetes.io/session-cookie-expires: "172800" + # nginx.ingress.kubernetes.io/session-cookie-max-age: "172800" # old way of setting the ingress class + {{- if semverCompare "<1.18.0" .Capabilities.KubeVersion.GitVersion }} kubernetes.io/ingress.class: {{ .Values.ingress.className }} - {{- end -}} - nginx.ingress.kubernetes.io/affinity: "cookie" - nginx.ingress.kubernetes.io/session-cookie-name: "lm-api-back-end" - nginx.ingress.kubernetes.io/session-cookie-expires: "172800" - nginx.ingress.kubernetes.io/session-cookie-max-age: "172800" - {{- end }} + {{- end }} spec: + {{- if semverCompare ">=1.18.0" .Capabilities.KubeVersion.GitVersion }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} {{- if .Values.ingress.tls }} tls: - {{- range .Values.ingress.tls }} + {{- range .Values.ingress.tls }} - hosts: - {{- range .hosts }} + {{- range .hosts }} - {{ . | quote }} - {{- end }} + {{- end }} secretName: {{ .secretName }} - {{- end }} + {{- end }} {{- end }} rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ . | quote }} - pathType: Prefix - backend: - service: - name: {{ $fullName }}-nginx - port: - number: 8080 + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ . | quote }} + {{- if $serviceNewStyle }} + pathType: Prefix + {{- end }} + backend: + {{- if $serviceNewStyle }} + service: + name: {{$fullname}}-nginx + port: + name: http + {{- else }} + serviceName: {{$fullname}}-nginx + servicePort: http {{- end }} - {{- end }} + {{- end }} {{- end }} +{{- end }} \ No newline at end of file From da4d1c1228f6bbac8d34063d40c7d852c2f2c013 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 25 Jul 2023 15:28:49 +0200 Subject: [PATCH 71/90] fix(k8s): missing scheme on external server url --- k8s/templates/settings.secret.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/templates/settings.secret.yaml b/k8s/templates/settings.secret.yaml index 8e372f33..8c19c8c0 100644 --- a/k8s/templates/settings.secret.yaml +++ b/k8s/templates/settings.secret.yaml @@ -17,7 +17,7 @@ stringData: {{- if .Values.externalServerName }} # The name and port number of the server (e.g.: 'lm.local:8000'), # used as base_url on all the links returned by the API - EXTERNAL_SERVER_URL={{ .Values.externalServerName }} + EXTERNAL_SERVER_URL=https://{{ .Values.externalServerName }} {{- end }} # Base URL of the LifeMonitor web app associated with this back-end instance From c6021094e0a5fec03bae165fbbcc0a54d48efcd4 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 26 Jul 2023 15:16:38 +0200 Subject: [PATCH 72/90] build(k8s): fix missing namespace on PVC --- k8s/templates/data-backup.pvc.yaml | 1 + k8s/templates/data-logs.pvc.yaml | 1 + k8s/templates/data-workflows.pvc.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/k8s/templates/data-backup.pvc.yaml b/k8s/templates/data-backup.pvc.yaml index 1f3994f6..2c3cdcd1 100644 --- a/k8s/templates/data-backup.pvc.yaml +++ b/k8s/templates/data-backup.pvc.yaml @@ -4,6 +4,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: {{ include "chart.lifemonitor.data.backup" . }} + namespace: {{ .Release.Namespace }} spec: storageClassName: {{ .Values.readWriteManyStorageClass }} accessModes: diff --git a/k8s/templates/data-logs.pvc.yaml b/k8s/templates/data-logs.pvc.yaml index 098dc472..3475af02 100644 --- a/k8s/templates/data-logs.pvc.yaml +++ b/k8s/templates/data-logs.pvc.yaml @@ -4,6 +4,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: {{ include "chart.lifemonitor.data.logs" . }} + namespace: {{ .Release.Namespace }} spec: storageClassName: {{ .Values.readWriteManyStorageClass }} accessModes: diff --git a/k8s/templates/data-workflows.pvc.yaml b/k8s/templates/data-workflows.pvc.yaml index 710901b9..75c084f8 100644 --- a/k8s/templates/data-workflows.pvc.yaml +++ b/k8s/templates/data-workflows.pvc.yaml @@ -4,6 +4,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: {{ include "chart.lifemonitor.data.workflows" . }} + namespace: {{ .Release.Namespace }} spec: storageClassName: {{ .Values.readWriteManyStorageClass }} accessModes: From 19658161e4df155032f7e64f331dd5c460677bfc Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 26 Jul 2023 15:17:35 +0200 Subject: [PATCH 73/90] build(k8s): fix reference to storage class --- k8s/templates/data-backup.pvc.yaml | 2 +- k8s/templates/data-logs.pvc.yaml | 2 +- k8s/templates/data-workflows.pvc.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/templates/data-backup.pvc.yaml b/k8s/templates/data-backup.pvc.yaml index 2c3cdcd1..2ab35b82 100644 --- a/k8s/templates/data-backup.pvc.yaml +++ b/k8s/templates/data-backup.pvc.yaml @@ -6,7 +6,7 @@ metadata: name: {{ include "chart.lifemonitor.data.backup" . }} namespace: {{ .Release.Namespace }} spec: - storageClassName: {{ .Values.readWriteManyStorageClass }} + storageClassName: {{ .Values.global.readWriteManyStorageClass }} accessModes: - ReadWriteMany resources: diff --git a/k8s/templates/data-logs.pvc.yaml b/k8s/templates/data-logs.pvc.yaml index 3475af02..7abd1a9c 100644 --- a/k8s/templates/data-logs.pvc.yaml +++ b/k8s/templates/data-logs.pvc.yaml @@ -6,7 +6,7 @@ metadata: name: {{ include "chart.lifemonitor.data.logs" . }} namespace: {{ .Release.Namespace }} spec: - storageClassName: {{ .Values.readWriteManyStorageClass }} + storageClassName: {{ .Values.global.readWriteManyStorageClass }} accessModes: - ReadWriteMany resources: diff --git a/k8s/templates/data-workflows.pvc.yaml b/k8s/templates/data-workflows.pvc.yaml index 75c084f8..d2092fa0 100644 --- a/k8s/templates/data-workflows.pvc.yaml +++ b/k8s/templates/data-workflows.pvc.yaml @@ -6,7 +6,7 @@ metadata: name: {{ include "chart.lifemonitor.data.workflows" . }} namespace: {{ .Release.Namespace }} spec: - storageClassName: {{ .Values.readWriteManyStorageClass }} + storageClassName: {{ .Values.global.readWriteManyStorageClass }} accessModes: - ReadWriteMany resources: From f6eaf2efe24f37a976277f095a7c562b26d8a206 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 5 Sep 2023 16:58:42 +0200 Subject: [PATCH 74/90] feat(k8s): allow to expose prometheus metrics --- k8s/templates/backend.deployment.yaml | 10 ++++++---- k8s/templates/worker.deployment.yaml | 2 ++ k8s/templates/wss.deployment.yaml | 10 ++++++---- k8s/values.yaml | 2 ++ 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/k8s/templates/backend.deployment.yaml b/k8s/templates/backend.deployment.yaml index 041f6a89..8695d3d6 100644 --- a/k8s/templates/backend.deployment.yaml +++ b/k8s/templates/backend.deployment.yaml @@ -23,10 +23,12 @@ spec: labels: {{- include "chart.selectorLabels" . | nindent 8 }} app.kubernetes.io/component: backend - # prometheus.io/scrape: 'true' - # prometheus.io/path: 'metrics' - # prometheus.io/port: '9090' - # prometheus.io/scheme: 'http' + {{- if $.Values.monitoring.enabled }} + prometheus.io/scrape: 'true' + prometheus.io/path: 'metrics' + prometheus.io/port: '9090' + prometheus.io/scheme: 'http' + {{- end }} spec: {{- with .Values.lifemonitor.imagePullSecrets }} imagePullSecrets: diff --git a/k8s/templates/worker.deployment.yaml b/k8s/templates/worker.deployment.yaml index 67a4acc6..f5ff41e8 100644 --- a/k8s/templates/worker.deployment.yaml +++ b/k8s/templates/worker.deployment.yaml @@ -24,10 +24,12 @@ spec: {{- end }} labels: {{- include "chart.selectorLabels" $ | nindent 8 }} + {{- if $.Values.monitoring.enabled }} prometheus.io/scrape: 'true' prometheus.io/path: 'metrics' prometheus.io/port: '9191' prometheus.io/scheme: 'http' + {{- end }} spec: {{- with $.Values.worker.imagePullSecrets }} imagePullSecrets: diff --git a/k8s/templates/wss.deployment.yaml b/k8s/templates/wss.deployment.yaml index 074a41f5..a127f74e 100644 --- a/k8s/templates/wss.deployment.yaml +++ b/k8s/templates/wss.deployment.yaml @@ -23,10 +23,12 @@ spec: labels: {{- include "chart.selectorLabels" . | nindent 8 }} app.kubernetes.io/component: wss - # prometheus.io/scrape: 'true' - # prometheus.io/path: 'metrics' - # prometheus.io/port: '9090' - # prometheus.io/scheme: 'http' + {{- if $.Values.monitoring.enabled }} + prometheus.io/scrape: 'true' + prometheus.io/path: 'metrics' + prometheus.io/port: '9090' + prometheus.io/scheme: 'http' + {{- end }} spec: {{- with .Values.lifemonitor.imagePullSecrets }} imagePullSecrets: diff --git a/k8s/values.yaml b/k8s/values.yaml index 0039c075..ecc4a339 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -130,6 +130,8 @@ backup: path: /user/home/lm-backups tls: true +monitoring: + enabled: false lifemonitor: replicaCount: 1 From 20cf219cfa5e3221e62ccd6fe45b6dd5512dc49e Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 5 Sep 2023 16:59:16 +0200 Subject: [PATCH 75/90] style(k8s): remove blanks --- k8s/templates/service.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/k8s/templates/service.yaml b/k8s/templates/service.yaml index 0aec5968..f8a8aca6 100644 --- a/k8s/templates/service.yaml +++ b/k8s/templates/service.yaml @@ -33,6 +33,3 @@ spec: selector: {{- include "chart.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: wss - ---- - From de470708d713f2ea03a772f24c8043db57529efc Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 5 Sep 2023 17:00:08 +0200 Subject: [PATCH 76/90] feat(k8s): convert back-end deployment to statefulset --- k8s/templates/backend.deployment.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/k8s/templates/backend.deployment.yaml b/k8s/templates/backend.deployment.yaml index 8695d3d6..2ea2622c 100644 --- a/k8s/templates/backend.deployment.yaml +++ b/k8s/templates/backend.deployment.yaml @@ -1,5 +1,5 @@ apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: {{ include "chart.fullname" . }}-backend labels: @@ -8,6 +8,7 @@ metadata: spec: {{- if not .Values.lifemonitor.autoscaling.enabled }} replicas: {{ .Values.lifemonitor.replicaCount }} + serviceName: "lifemonitor-backend" {{- end }} selector: matchLabels: From 0c5e837c6e0fe6e6071157f47067f1ad047d5e94 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 5 Sep 2023 17:00:50 +0200 Subject: [PATCH 77/90] refactor(k8s): rename containers of the main pod --- k8s/templates/backend.deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/templates/backend.deployment.yaml b/k8s/templates/backend.deployment.yaml index 2ea2622c..17ac699b 100644 --- a/k8s/templates/backend.deployment.yaml +++ b/k8s/templates/backend.deployment.yaml @@ -39,7 +39,7 @@ spec: securityContext: {{- toYaml .Values.lifemonitor.podSecurityContext | nindent 8 }} initContainers: - - name: init + - name: init-backend securityContext: {{- toYaml .Values.lifemonitor.securityContext | nindent 12 }} image: {{ include "chart.lifemonitor.image" . }} @@ -51,7 +51,7 @@ spec: volumeMounts: {{- include "lifemonitor.common-volume-mounts" . | nindent 12 }} containers: - - name: app + - name: backend securityContext: {{- toYaml .Values.lifemonitor.securityContext | nindent 12 }} image: {{ include "chart.lifemonitor.image" . }} From 746dc32ae24456349e778bd60c075f365944be0c Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 5 Sep 2023 17:02:08 +0200 Subject: [PATCH 78/90] refactor(k8s): use multiline command --- k8s/templates/backend.deployment.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/k8s/templates/backend.deployment.yaml b/k8s/templates/backend.deployment.yaml index 17ac699b..3e94f0d5 100644 --- a/k8s/templates/backend.deployment.yaml +++ b/k8s/templates/backend.deployment.yaml @@ -44,8 +44,12 @@ spec: {{- toYaml .Values.lifemonitor.securityContext | nindent 12 }} image: {{ include "chart.lifemonitor.image" . }} imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} - command: ["/bin/sh","-c"] - args: ["wait-for-redis.sh && wait-for-postgres.sh && ./lm-admin db wait-for-db"] + command: + - "/bin/sh" + - "-c" + - | + wait-for-redis.sh && wait-for-postgres.sh + ./lm-admin db wait-for-db env: {{- include "lifemonitor.common-env" . | nindent 12 }} volumeMounts: From a0f0a31e5fc5bfdc10100174e7a3224820fa2b7b Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 5 Sep 2023 17:04:38 +0200 Subject: [PATCH 79/90] feat(k8s): add extra headless svc and svc monitors --- k8s/templates/monitoring.service.yaml | 94 +++++++++++++++++++++++++++ k8s/values.yaml | 5 ++ 2 files changed, 99 insertions(+) create mode 100644 k8s/templates/monitoring.service.yaml diff --git a/k8s/templates/monitoring.service.yaml b/k8s/templates/monitoring.service.yaml new file mode 100644 index 00000000..34973b7e --- /dev/null +++ b/k8s/templates/monitoring.service.yaml @@ -0,0 +1,94 @@ +{{- if .Values.monitoring.enabled -}} +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: '9090' + labels: + {{- include "chart.labels" . | nindent 4 }} + app.kubernetes.io/component: backend-metrics-exporter + name: {{ include "chart.fullname" . }}-backend-metrics-headless + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + selector: + app.kubernetes.io/component: backend + ports: + - name: metrics + protocol: TCP + port: 9090 + targetPort: 9090 +--- + +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: '9090' + labels: + {{- include "chart.labels" . | nindent 4 }} + app.kubernetes.io/component: backend-proxy-metrics-exporter + name: {{ include "chart.fullname" . }}-backend-proxy-metrics-headless + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + selector: + app.kubernetes.io/name: nginx + app.kubernetes.io/instance: api + ports: + - name: metrics + protocol: TCP + port: 9090 + targetPort: 9090 + +--- + +{{- if .Values.monitoring.service_monitor.enabled -}} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "chart.fullname" . }}-lifemonitor-backend-metrics-servicemonitor + # same namespace that Prometheus is running in + namespace: {{ .Values.monitoring.prometheus.namespace }} + labels: + app: {{ include "chart.fullname" . }}-backend + release: prometheus-stack +spec: + selector: + matchLabels: + app.kubernetes.io/component: backend-metrics-exporter + endpoints: + - path: /metrics + port: metrics + interval: 15s + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} # namespace where the app is running + +--- + +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "chart.fullname" . }}-lifemonitor-backend-proxy-metrics-servicemonitor + # same namespace that Prometheus is running in + namespace: {{ .Values.monitoring.prometheus.namespace }} + labels: + app: {{ include "chart.fullname" . }}-backend + release: prometheus-stack +spec: + selector: + matchLabels: + app.kubernetes.io/component: backend-proxy-metrics-exporter + endpoints: + - path: /metrics + port: metrics + interval: 15s + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} # namespace where the app is running +{{- end -}} + +{{- end -}} \ No newline at end of file diff --git a/k8s/values.yaml b/k8s/values.yaml index ecc4a339..9954db97 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -132,6 +132,11 @@ backup: monitoring: enabled: false + service_monitor: + enabled: false + prometheus: + namespace: kube-prometheus-stack + lifemonitor: replicaCount: 1 From abb72a21bf9a71ae5285e25410ae6a166e4f6c5a Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 5 Sep 2023 17:06:08 +0200 Subject: [PATCH 80/90] refactor(k8s): init empty dir to store pod logs --- k8s/templates/_helpers.tpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/k8s/templates/_helpers.tpl b/k8s/templates/_helpers.tpl index 78c32cf8..0cc0efba 100644 --- a/k8s/templates/_helpers.tpl +++ b/k8s/templates/_helpers.tpl @@ -145,6 +145,8 @@ Define volumes shared by some pods. - name: lifemonitor-settings secret: secretName: {{ include "chart.fullname" . }}-settings +- name: lifemonitor-logs + emptyDir: {} - name: lifemonitor-data persistentVolumeClaim: claimName: data-{{- .Release.Name -}}-workflows From 862fce161381eaff4266f620c6ee58d2ecdf7015 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 5 Sep 2023 17:07:42 +0200 Subject: [PATCH 81/90] style(k8s): remove blank --- k8s/templates/_helpers.tpl | 1 - 1 file changed, 1 deletion(-) diff --git a/k8s/templates/_helpers.tpl b/k8s/templates/_helpers.tpl index 0cc0efba..6cf6c56e 100644 --- a/k8s/templates/_helpers.tpl +++ b/k8s/templates/_helpers.tpl @@ -49,7 +49,6 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* - Define lifemonitor image */}} {{- define "chart.lifemonitor.image" -}} From 79a6a2c868c0f95f4736e829e6949b5784c6f917 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 5 Sep 2023 17:08:25 +0200 Subject: [PATCH 82/90] chore(k8s): add extra helpers for ghApp keys --- k8s/templates/_helpers.tpl | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/k8s/templates/_helpers.tpl b/k8s/templates/_helpers.tpl index 6cf6c56e..905f6744 100644 --- a/k8s/templates/_helpers.tpl +++ b/k8s/templates/_helpers.tpl @@ -194,4 +194,21 @@ Generate certificates for the LifeMonitor Api Server . {{- $cert := genSignedCert ( include "chart.name" . ) nil $altNames 365 $ca -}} tls.crt: {{ $cert.Cert | b64enc }} tls.key: {{ $cert.Key | b64enc }} -{{- end -}} \ No newline at end of file +{{- end -}} + + +{{/* +Define lifemonitor GithubApp secret name +*/}} +{{- define "chart.lifemonitor.githubApp.key" -}} +{{- printf "%s-ghapp-key" .Release.Name }} +{{- end }} + +{{/* +Read and encode the GitHub App private key. +*/}} +{{- define "lifemonitor.githubApp.readPrivateKey" -}} +{{- $fileContent := $.Files.Get .Values.integrations.github.private_key.path -}} +{{- $base64Content := $fileContent | b64enc -}} +{{- printf "%s" $base64Content -}} +{{- end -}} From 56489558052eb74925a9596b2df7b99e746c56e5 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 5 Sep 2023 17:11:21 +0200 Subject: [PATCH 83/90] chore(k8s): bump chart version --- k8s/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/Chart.yaml b/k8s/Chart.yaml index 68df7269..0632158d 100644 --- a/k8s/Chart.yaml +++ b/k8s/Chart.yaml @@ -7,7 +7,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.8.0 +version: 0.10.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -17,7 +17,7 @@ appVersion: 0.11.4 # Chart dependencies dependencies: - name: nginx - version: 8.8.4 + version: 13.2.28 repository: https://charts.bitnami.com/bitnami - name: postgresql version: 10.1.1 From a3131e73efeb716a333c41d6e4401e9e61f5aa54 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 5 Sep 2023 17:18:51 +0200 Subject: [PATCH 84/90] feat(k8s): add template of grafana dashboard --- utils/grafana/LifeMonitorDashboard.json | 2254 +++++++++++++++++++++++ 1 file changed, 2254 insertions(+) create mode 100644 utils/grafana/LifeMonitorDashboard.json diff --git a/utils/grafana/LifeMonitorDashboard.json b/utils/grafana/LifeMonitorDashboard.json new file mode 100644 index 00000000..97665690 --- /dev/null +++ b/utils/grafana/LifeMonitorDashboard.json @@ -0,0 +1,2254 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "panel", + "id": "geomap", + "name": "Geomap", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.1.1" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 22, + "panels": [], + "title": "LifeMonitor Objects", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of users registered on this LifeMonitor instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "neutral": 38 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 18, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "sum(lifemonitor_api_users)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Users", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of workflows registered on this LifeMonitor instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 20, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "sum(lifemonitor_api_workflows)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Workflows", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of workflow versions registered on this LifeMonitor instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 24, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "sum(lifemonitor_api_workflow_versions)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Workflow Versions", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Workflow registries registered on the LifeMonitor instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 26, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "sum(lifemonitor_api_workflow_registries)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Workflow Registries", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of workflow suites registered on the LifeMonitor instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 28, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "sum(lifemonitor_api_workflow_suites)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Workflow Suites", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of workflow test instances registered on the LifeMonitor instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 30, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "sum(lifemonitor_api_workflow_test_instances)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Workflow Test Instances", + "type": "gauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 45, + "panels": [], + "title": "LifeMonitor WebApp Active Clients", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Current Users on the Web App", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 0, + "y": 8 + }, + "id": 36, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(lifemonitor_webapp_http_websocket_connections_total)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Active Web App Clients", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 19, + "x": 5, + "y": 8 + }, + "id": 52, + "options": { + "basemap": { + "config": {}, + "name": "Layer 0", + "type": "default" + }, + "controls": { + "mouseWheelZoom": true, + "showAttribution": true, + "showDebug": true, + "showMeasure": false, + "showScale": true, + "showZoom": true + }, + "layers": [ + { + "config": { + "showLegend": true, + "style": { + "color": { + "field": "Last", + "fixed": "dark-green" + }, + "opacity": 0.4, + "rotation": { + "field": "Last", + "fixed": 0, + "max": 360, + "min": -360, + "mode": "mod" + }, + "size": { + "field": "Total", + "fixed": 5, + "max": 15, + "min": 2 + }, + "symbol": { + "fixed": "img/icons/marker/circle.svg", + "mode": "fixed" + }, + "text": { + "field": "Last", + "fixed": "", + "mode": "field" + }, + "textConfig": { + "fontSize": 12, + "offsetX": 10, + "offsetY": 0, + "textAlign": "left", + "textBaseline": "middle" + } + } + }, + "filterData": { + "id": "byRefId", + "options": "A" + }, + "location": { + "latitude": "lifemonitor_api_proxy_http_requests_total", + "mode": "auto" + }, + "name": "Web Clients", + "tooltip": true, + "type": "markers" + } + ], + "tooltip": { + "mode": "details" + }, + "view": { + "allLayers": true, + "id": "zero", + "lat": 0, + "lon": 0, + "zoom": 1 + } + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "lifemonitor_webapp_http_websocket_connections_total", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Location of Active Web App Clients", + "transformations": [ + { + "id": "reduce", + "options": { + "includeTimeField": false, + "labelsToFields": true, + "mode": "seriesToRows", + "reducers": [ + "last" + ] + } + } + ], + "type": "geomap" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 44, + "panels": [], + "title": "Rate of LifeMonitor WebApp HTTP Connections to Back-end API", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 7, + "x": 0, + "y": 17 + }, + "id": 38, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(client_country_long) (rate(lifemonitor_webapp_http_requests_total[30s]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Requests by Country", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "status=200": { + "color": "purple", + "index": 0, + "text": "OK" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 5, + "x": 7, + "y": 17 + }, + "id": 39, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(lifemonitor_webapp_http_connections{status=\"200\"}[30s])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Status 200", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 12, + "y": 17 + }, + "id": 40, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(rate(lifemonitor_webapp_http_connections{status=~\"3[0-9][0-9]\"}[30s]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Status ~= 3**", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 16, + "y": 17 + }, + "id": 42, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(rate(lifemonitor_webapp_http_connections{status=~\"4[0-9][0-9]\"}[30s]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Status ~= 4**", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 20, + "y": 17 + }, + "id": 43, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(rate(lifemonitor_webapp_http_connections{status=~\"5[0-9][0-9]\"}[30s]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Status ~= 5**", + "transformations": [], + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 48, + "panels": [], + "title": "WebApp Http Connections to Back-end API by status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#5f675e", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 41, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(status) (lifemonitor_webapp_http_connections)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "WebApp HTTP Connections 200 status", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 53, + "panels": [], + "title": "Direct API Back-end Requests", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 34 + }, + "id": 49, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "increase(lifemonitor_api_proxy_http_active_connections_number[30s])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Active HTTP Connections", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 34 + }, + "id": 50, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "increase(lifemonitor_api_proxy_http_upstream_connections_total[30s])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Active Upstream Connections", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Active Clients By Country", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 34 + }, + "id": 47, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(client_country_long) (increase(lifemonitor_api_proxy_http_requests_total[30s]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Active Requests by Country", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Active Clients By Country", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 34 + }, + "id": 51, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(client_country_long) (lifemonitor_api_proxy_http_requests_total)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "# Requests by Country", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 46, + "panels": [], + "title": "LifeMonitor API Backend Requests", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "HTTP 500" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#bf1b00", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 13, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "$$hashKey": "object:140", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "increase(lifemonitor_api_http_request_total[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "HTTP {{ status }}", + "range": true, + "refId": "A" + } + ], + "title": "Total requests per minute", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "/workflows" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 2, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "$$hashKey": "object:214", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(lifemonitor_api_http_request_duration_seconds_count{status=\"200\"}[30s])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "range": true, + "refId": "A" + } + ], + "title": "Requests per second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#c15c17", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 4, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "$$hashKey": "object:766", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(lifemonitor_api_http_request_duration_seconds_count{status!=\"200\"}[30s]))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "errors", + "range": true, + "refId": "A" + } + ], + "title": "Errors per second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 71 + }, + "id": 6, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "$$hashKey": "object:146", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(lifemonitor_api_http_request_duration_seconds_sum{status=\"200\"}[30s])\n/\nrate(lifemonitor_api_http_request_duration_seconds_count{status=\"200\"}[30s])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "range": true, + "refId": "A" + } + ], + "title": "Average response time [30s]", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 80 + }, + "id": 11, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "$$hashKey": "object:1079", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "increase(lifemonitor_api_http_request_duration_seconds_bucket{status=\"200\",le=\"0.25\"}[30s]) \n/ ignoring (le) increase(lifemonitor_api_http_request_duration_seconds_count{status=\"200\"}[30s])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "title": "Requests under 250ms", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 91 + }, + "id": 15, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "$$hashKey": "object:426", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.5, rate(lifemonitor_api_http_request_duration_seconds_bucket{status=\"200\"}[30s]))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "range": true, + "refId": "A" + } + ], + "title": "Request duration [s] - p50", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 102 + }, + "id": 16, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "$$hashKey": "object:426", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.9, rate(lifemonitor_api_http_request_duration_seconds_bucket{status=\"200\"}[30s]))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "range": true, + "refId": "A" + } + ], + "title": "Request duration [s] - p90", + "type": "timeseries" + } + ], + "refresh": "auto", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "3s" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "Europe/Rome", + "title": "LifeMonitor Dashboard", + "uid": "_eX4mpl3", + "version": 12, + "weekStart": "" +} \ No newline at end of file From a2e4ea01d6aa178a6fc2729ff9e397d23b0f7b0c Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 5 Sep 2023 17:37:09 +0200 Subject: [PATCH 85/90] style: fix flake8 issue E231 --- tests/integration/api/services/test_services.py | 4 ++-- tests/unit/api/controllers/test_workflows.py | 2 +- tests/unit/auth/models/test_users.py | 4 ++-- tests/unit/schemas/test_lm_schema_validator.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/integration/api/services/test_services.py b/tests/integration/api/services/test_services.py index 3369cc20..045730f9 100644 --- a/tests/integration/api/services/test_services.py +++ b/tests/integration/api/services/test_services.py @@ -44,7 +44,7 @@ def test_workflow_registration(app_client, user1, valid_workflow): logger.debug("Registered workflow: %r", workflow) assert workflow is not None, "workflow must be not None" assert isinstance(workflow, models.WorkflowVersion), "Object is not an instance of WorkflowVersion" - assert (str(workflow.workflow.uuid), workflow.version) == (w['uuid'], w['version']),\ + assert (str(workflow.workflow.uuid), workflow.version) == (w['uuid'], w['version']), \ "Unexpected workflow ID" assert len(models.Workflow.all()) == 1, "Unexpected number of workflows" assert len(models.WorkflowVersion.all()) == 1, "Unexpected number of workflow_versions" @@ -134,7 +134,7 @@ def test_workflow_registry_generic_link(app_client, user1): # , valid_workflow) _, workflow = utils.register_workflow(user1, w) assert workflow is not None, "workflow must be not None" assert isinstance(workflow, models.WorkflowVersion), "Object is not an instance of WorkflowVersion" - assert (workflow.workflow.uuid, workflow.version) == (w['uuid'], w['version']),\ + assert (workflow.workflow.uuid, workflow.version) == (w['uuid'], w['version']), \ "Unexpected workflow ID" # assert workflow.external_id is not None, "External ID must be computed if not provided" # assert workflow.external_id == w["external_id"], "Invalid external ID" diff --git a/tests/unit/api/controllers/test_workflows.py b/tests/unit/api/controllers/test_workflows.py index 583c42e5..ede42d79 100644 --- a/tests/unit/api/controllers/test_workflows.py +++ b/tests/unit/api/controllers/test_workflows.py @@ -239,7 +239,7 @@ def test_post_workflow_by_registry_error_submitter_not_found(m, request_context, with pytest.raises(lm_exceptions.NotAuthorizedException) as ex: controllers.workflows_post(body=data) assert messages.no_user_oauth_identity_on_registry \ - .format(data["submitter_id"], mock_registry.name) in ex.exconly(True),\ + .format(data["submitter_id"], mock_registry.name) in ex.exconly(True), \ "Unexpected error message" diff --git a/tests/unit/auth/models/test_users.py b/tests/unit/auth/models/test_users.py index d79a1a15..9c597e55 100644 --- a/tests/unit/auth/models/test_users.py +++ b/tests/unit/auth/models/test_users.py @@ -35,7 +35,7 @@ def test_identity(app_client, user1, client_credentials_registry): assert user.current_identity is not None, "Current identity should not be empty" identity = user.current_identity[client_credentials_registry.name] - assert identity,\ + assert identity, \ f"Current identity should contain an identity issued by the provider {client_credentials_registry.name}" assert user.current_identity[client_credentials_registry.name].provider == client_credentials_registry.server_credentials, \ "Unexpected identity provider" @@ -44,7 +44,7 @@ def test_identity(app_client, user1, client_credentials_registry): logger.debug(serialization) assert "identities" in serialization, \ "Unable to find the property 'identity' on the serialized user" - assert serialization['identities'][client_credentials_registry.name]['provider']['name'] == client_credentials_registry.name,\ + assert serialization['identities'][client_credentials_registry.name]['provider']['name'] == client_credentials_registry.name, \ "Invalid provider" diff --git a/tests/unit/schemas/test_lm_schema_validator.py b/tests/unit/schemas/test_lm_schema_validator.py index 214f3315..120735d8 100644 --- a/tests/unit/schemas/test_lm_schema_validator.py +++ b/tests/unit/schemas/test_lm_schema_validator.py @@ -87,7 +87,7 @@ def test_default_update_registries_of_branch(data, schema): assert result is not None, "Result should be empty" assert result.valid is True, "Data should be valid" print(json.dumps(result.output_data, indent=2)) - assert 'update_registries' in result.output_data['push']['branches'][0],\ + assert 'update_registries' in result.output_data['push']['branches'][0], \ "update_registries should be automatically initialized" assert result.output_data['push']['branches'][0]['update_registries'] == [], \ "update_registries should be automatically initialized with default values" From 0891a308154474c40339973f03945334ae0d3778 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 5 Sep 2023 17:39:14 +0200 Subject: [PATCH 86/90] fix: disable unused property redefinition --- lifemonitor/api/models/workflows.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lifemonitor/api/models/workflows.py b/lifemonitor/api/models/workflows.py index e927a1dd..901bb4a0 100644 --- a/lifemonitor/api/models/workflows.py +++ b/lifemonitor/api/models/workflows.py @@ -294,9 +294,9 @@ def authorizations(self): auths.append(auth) return auths - @hybrid_property - def roc_link(self) -> str: - return self.uri + # @hybrid_property + # def roc_link(self) -> str: + # return self.uri @property def workflow_name(self) -> str: From f78a9570c6c5194691ce6c35ae0f3a84f44f931a Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 6 Sep 2023 12:26:02 +0200 Subject: [PATCH 87/90] fix(model): remove redundant association-proxy --- lifemonitor/api/models/workflows.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lifemonitor/api/models/workflows.py b/lifemonitor/api/models/workflows.py index 901bb4a0..d8eec55c 100644 --- a/lifemonitor/api/models/workflows.py +++ b/lifemonitor/api/models/workflows.py @@ -23,7 +23,6 @@ import logging from typing import List, Optional, Set, Union -from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import aliased from sqlalchemy.orm.collections import (MappedCollection, @@ -215,8 +214,6 @@ class WorkflowVersion(ROCrate): backref=db.backref("workflows", cascade="all, delete-orphan", collection_class=WorkflowVersionCollection)) - roc_link = association_proxy('ro_crate', 'uri') - __mapper_args__ = { 'polymorphic_identity': 'workflow_version' } @@ -294,9 +291,9 @@ def authorizations(self): auths.append(auth) return auths - # @hybrid_property - # def roc_link(self) -> str: - # return self.uri + @hybrid_property + def roc_link(self) -> str: + return self.uri @property def workflow_name(self) -> str: From df0318300dad7904aa0343f002bf5415f37fdf0f Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 5 Sep 2023 17:44:20 +0200 Subject: [PATCH 88/90] build(pip): bump uWSGI to 2.0.22 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8388cab7..38ad1c14 100644 --- a/requirements.txt +++ b/requirements.txt @@ -55,7 +55,7 @@ urllib3==1.26.11 flask-socketio==5.3.4 gevent-websocket==0.10.1 eventlet==0.33.3 -uwsgi==2.0.21 +uwsgi==2.0.22 # watchdog version 2.3 introduces a change that breaks Flask in development mode. # Don't use watchdog > 2.2.1 until up update Flask to something >= 2.2 watchdog==2.2.1 From 8e89fb10e7a7780994530ff2461ce30aace814e0 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 5 Sep 2023 17:45:45 +0200 Subject: [PATCH 89/90] build(pip): bump cryptography to 41.0.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 38ad1c14..1c0cfcea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ boto3~=1.24.80 Bcrypt-Flask==1.0.2 giturlparse~=0.10.0 click-option-group~=0.5.5 -cryptography==41.0.2 +cryptography>=41.0.3 dnspython==2.2.1 flask-cors==3.0.10 flask-marshmallow~=0.14.0 From 0068d8ff591dde3536adda581b15ef258a3015fa Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 5 Sep 2023 17:48:48 +0200 Subject: [PATCH 90/90] chore: bump app version to 0.11.5 --- k8s/Chart.yaml | 2 +- lifemonitor/static/src/package.json | 2 +- specs/api.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/k8s/Chart.yaml b/k8s/Chart.yaml index 0632158d..f7e65bfd 100644 --- a/k8s/Chart.yaml +++ b/k8s/Chart.yaml @@ -12,7 +12,7 @@ version: 0.10.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 0.11.4 +appVersion: 0.11.5 # Chart dependencies dependencies: diff --git a/lifemonitor/static/src/package.json b/lifemonitor/static/src/package.json index 41f4e8ed..7681507e 100644 --- a/lifemonitor/static/src/package.json +++ b/lifemonitor/static/src/package.json @@ -1,7 +1,7 @@ { "name": "lifemonitor", "description": "Workflow Testing Service", - "version": "0.11.4", + "version": "0.11.5", "license": "MIT", "author": "CRS4", "main": "../dist/js/lifemonitor.min.js", diff --git a/specs/api.yaml b/specs/api.yaml index 8a729a94..62938e47 100644 --- a/specs/api.yaml +++ b/specs/api.yaml @@ -3,7 +3,7 @@ openapi: "3.0.0" info: - version: "0.11.4" + version: "0.11.5" title: "Life Monitor API" description: | *Workflow sustainability service* @@ -18,7 +18,7 @@ info: servers: - url: / description: > - Version 0.11.4 of API. + Version 0.11.5 of API. tags: - name: GitHub Integration