From 3920673eafdb8a91a5d39f3c8f6241dd9c422e4f Mon Sep 17 00:00:00 2001 From: why-not-try-calmer <mrnycticorax@gmail.com> Date: Thu, 8 Jun 2023 18:39:53 +0200 Subject: [PATCH] Adding support for pyproject.toml files - pyproject.toml are becoming the standard nowadays for packaging Python applications so I felt it would be nice to have. - did some DRY-cleaning, moving the logic initializing the argparse Parameters to a stand-alone top level function. It could be an instance method of Parameters too, it's easy to refactor from one to the other. - added tests leveraging the two aforementioend changes - reduced time complexity when collecting metadata - sprinkled some type hints here and there README --- collecting metadata: O(n) from O(n^2) --- README.md | 17 ++++-- qgis_plugin_CI_testing/qgissettingmanager | 2 +- qgispluginci/cli.py | 32 +--------- qgispluginci/parameters.py | 74 +++++++++++++++-------- qgispluginci/utils.py | 65 +++++++++++++++++++- requirements/base.txt | 1 + test/test_release.py | 38 ++++++++---- 7 files changed, 158 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index eccf534e..be28bb99 100644 --- a/README.md +++ b/README.md @@ -53,10 +53,19 @@ commands: ## Requirements -- The code is under a **git** repository (`git archive` is used to bundle the plugin) -- There is no uncommitted changes when doing a package/release (there is an option to allow this) -- A configuration at the top directory either in `.qgis-plugin-ci` or in `setup.cfg` with a `[qgis-plugin-ci]` section. -- The source files of the plugin are within a sub-directory. The name of this directory will be used for the zip file. +- The code is under a **git** repository (`git archive` is used to bundle the plugin). +- There is no uncommitted changes when doing a package/release (althought there is an option to bypass this requirement). +- A configuration at the top directory either in `.qgis-plugin-ci` or in `setup.cfg` or `pyproject.toml` with a `[qgis-plugin-ci]` section with the following fields: + - github_organization_slug (unless you're running from Travis CI) + - plugin_path + - project_slug (unless you're running from Travis CI) +- The source files of the plugin are within a sub-directory, with among others, a `metadata.txt` file with the following fields: + - description + - qgisMinimumVersion + - repository + - tracker + +See `parameters.py` for more parameters and details. Notice that the name of this directory will be used for the zip file. ## QRC and UI files diff --git a/qgis_plugin_CI_testing/qgissettingmanager b/qgis_plugin_CI_testing/qgissettingmanager index 9d9ec05b..565a0bcd 160000 --- a/qgis_plugin_CI_testing/qgissettingmanager +++ b/qgis_plugin_CI_testing/qgissettingmanager @@ -1 +1 @@ -Subproject commit 9d9ec05beb3f0462b5611190509063de6c0109d3 +Subproject commit 565a0bcd496ec3e42b3e39e06abb7cd24b8af48d diff --git a/qgispluginci/cli.py b/qgispluginci/cli.py index bcf9a868..877299aa 100755 --- a/qgispluginci/cli.py +++ b/qgispluginci/cli.py @@ -1,18 +1,12 @@ #!/usr/bin/env python3 - import argparse -import configparser import logging -import os from importlib.metadata import version -import yaml - from qgispluginci.changelog import ChangelogParser -from qgispluginci.exceptions import ConfigurationNotFound -from qgispluginci.parameters import Parameters from qgispluginci.release import release from qgispluginci.translation import Translation +from qgispluginci.utils import make_parameters __version__ = version("qgis-plugin-ci") __title__ = "QGISPluginCI" @@ -169,28 +163,8 @@ def cli(): exit_val = 0 - if os.path.isfile(".qgis-plugin-ci"): - # We read the .qgis-plugin-ci file - with open(".qgis-plugin-ci", encoding="utf8") as f: - arg_dict = yaml.safe_load(f) - else: - config = configparser.ConfigParser() - config.read("setup.cfg") - if "qgis-plugin-ci" in config.sections(): - # We read the setup.cfg file - arg_dict = dict(config.items("qgis-plugin-ci")) - else: - # We don't have either a .qgis-plugin-ci or a setup.cfg - if args.command == "changelog": - # but for the "changelog" sub command, the config file is not required, we can continue - arg_dict = dict() - else: - raise ConfigurationNotFound( - ".qgis-plugin-ci or setup.cfg with a 'qgis-plugin-ci' section have not been found." - ) - - parameters = Parameters(arg_dict) - + # Initialize Parameters + parameters = make_parameters(args) # CHANGELOG if args.command == "changelog": try: diff --git a/qgispluginci/parameters.py b/qgispluginci/parameters.py index f5d9e3a8..d90d7d5b 100644 --- a/qgispluginci/parameters.py +++ b/qgispluginci/parameters.py @@ -14,6 +14,7 @@ import os import re import sys +from typing import Any, Callable, Iterator, Optional, Tuple # 3rd party from slugify import slugify @@ -99,7 +100,9 @@ class Parameters: def __init__(self, definition: dict): self.plugin_path = definition.get("plugin_path") - self.plugin_name = self.__get_from_metadata("name") + + get_metadata = self.collect_metadata() + self.plugin_name = get_metadata("name") self.plugin_slug = slugify(self.plugin_name) self.project_slug = definition.get( "project_slug", @@ -146,22 +149,22 @@ def __init__(self, definition: dict): # This tool can be used outside of a QGIS plugin to read a changelog file return - self.author = self.__get_from_metadata("author", "") - self.description = self.__get_from_metadata("description") - self.qgis_minimum_version = self.__get_from_metadata("qgisMinimumVersion") - self.icon = self.__get_from_metadata("icon", "") - self.tags = self.__get_from_metadata("tags", "") - self.experimental = self.__get_from_metadata("experimental", False) - self.deprecated = self.__get_from_metadata("deprecated", False) - self.issue_tracker = self.__get_from_metadata("tracker") - self.homepage = self.__get_from_metadata("homepage", "") + self.author = get_metadata("author", "") + self.description = get_metadata("description") + self.qgis_minimum_version = get_metadata("qgisMinimumVersion") + self.icon = get_metadata("icon", "") + self.tags = get_metadata("tags", "") + self.experimental = get_metadata("experimental", False) + self.deprecated = get_metadata("deprecated", False) + self.issue_tracker = get_metadata("tracker") + self.homepage = get_metadata("homepage", "") if self.homepage == "": logger.warning( "Homepage is not given in the metadata. " "It is a mandatory information to publish " "the plugin on the QGIS official repository." ) - self.repository_url = self.__get_from_metadata("repository") + self.repository_url = get_metadata("repository") @staticmethod def archive_name( @@ -178,20 +181,43 @@ def archive_name( experimental = "-experimental" if experimental else "" return f"{plugin_name}{experimental}.{release_version}.zip" - def __get_from_metadata(self, key: str, default_value: any = None) -> str: - if not self.plugin_path: - return "" - + def collect_metadata(self) -> Callable[[str, Optional[Any]], Any]: + """ + Returns a closure capturing a Dict of metadata, allowing to retrieve one + value after the other while also iterating over the file once. + """ metadata_file = f"{self.plugin_path}/metadata.txt" - with open(metadata_file) as f: - for line in f: - m = re.match(rf"{key}\s*=\s*(.*)$", line) - if m: - return m.group(1) - if default_value is None: - logger.error(f"Mandatory key is missing in metadata: {key}") - sys.exit(1) - return default_value + metadata = {} + + with open(metadata_file) as fh: + for line in fh: + split = line.strip().split("=", 1) + if len(split) == 2: + metadata[split[0]] = split[1] + + def get_metadata(key: str, default_value: Optional[Any] = None) -> Any: + if not self.plugin_path: + return "" + + value = metadata.get(key, None) + if value: + return value + elif default_value is not None: + return default_value + else: + logger.error(f"Mandatory key is missing in metadata: {key}") + sys.exit(1) + + return get_metadata + + def __iter__(self) -> Iterator[Tuple[str, Any]]: + """Allows to represent attributes as dict, list, etc.""" + for k in vars(self): + yield k, self.__getattribute__(k) + + def __str__(self) -> str: + """Allows to represent instances as a string.""" + return str(dict(self)) # ############################################################################ diff --git a/qgispluginci/utils.py b/qgispluginci/utils.py index a53e2e7a..16ad6488 100644 --- a/qgispluginci/utils.py +++ b/qgispluginci/utils.py @@ -1,11 +1,17 @@ +import configparser import logging import os import re from math import floor from math import log as math_log from math import pow -from typing import Union +from typing import Any, Dict, Optional, Union +import toml +import yaml + +from qgispluginci.exceptions import ConfigurationNotFound +from qgispluginci.parameters import Parameters from qgispluginci.version_note import VersionNote # GLOBALS @@ -29,6 +35,63 @@ def configure_file(source_file: str, dest_file: str, replace: dict): f.write(content) +def make_parameters(args=None, config_file: Optional[str] = None) -> Parameters: + """ + Make a Dict from a config file or by exploring the filesystem + Accepts an argparse Namespace for backward compatibility. + """ + configuration_not_found = ConfigurationNotFound( + ".qgis-plugin-ci or setup.cfg or pyproject.toml with a 'qgis-plugin-ci' section have not been found." + ) + + def explore_config() -> Dict[str, Any]: + if os.path.isfile(".qgis-plugin-ci"): + # We read the .qgis-plugin-ci file + with open(".qgis-plugin-ci", encoding="utf8") as f: + arg_dict = yaml.safe_load(f) + elif os.path.isfile("pyproject.toml"): + # We read the pyproject.toml file + with open("pyproject.toml", encoding="utf8") as f: + arg_dict = toml.load(f) + else: + config = configparser.ConfigParser() + config.read("setup.cfg") + if "qgis-plugin-ci" in config.sections(): + # We read the setup.cfg file + arg_dict = dict(config.items("qgis-plugin-ci")) + else: + # We don't have either a .qgis-plugin-ci or a setup.cfg + if args and args.command == "changelog": + # but for the "changelog" sub command, the config file is not required, we can continue + arg_dict = dict() + else: + raise configuration_not_found + return arg_dict + + def load_config(filename: str) -> Dict[str, Any]: + if filename == "setup.cfg": + config = configparser.ConfigParser() + config.read(filename) + return dict(config.items("qgis-plugin-ci")) + + _, suffix = filename.rsplit(".", 1) + + with open(filename) as f: + if suffix == "toml": + return toml.load(f) + elif suffix in {"yaml", "yml"}: + return yaml.safe_load(f) + + raise configuration_not_found + + if config_file: + config_dict = load_config(config_file) + else: + config_dict = explore_config() + + return Parameters(config_dict) + + def convert_octets(octets: int) -> str: """Convert a mount of octets in readable size. diff --git a/requirements/base.txt b/requirements/base.txt index 2bd87a56..e4b00def 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -8,3 +8,4 @@ pyqt5ac>=1.2,<1.3 python-slugify>=4.0,<8.1 pyyaml>=5.4,<6.1 transifex-python>=3.2,<4.0 +toml>=0.10.2 diff --git a/test/test_release.py b/test/test_release.py index 4f6fb27f..74a5daf0 100644 --- a/test/test_release.py +++ b/test/test_release.py @@ -20,7 +20,7 @@ from qgispluginci.parameters import DASH_WARNING, Parameters from qgispluginci.release import release from qgispluginci.translation import Translation -from qgispluginci.utils import replace_in_file +from qgispluginci.utils import make_parameters, replace_in_file # Tests from .utils import can_skip_test @@ -31,9 +31,10 @@ class TestRelease(unittest.TestCase): def setUp(self): - with open(".qgis-plugin-ci", encoding="utf8") as f: - arg_dict = yaml.safe_load(f) - self.parameters = Parameters(arg_dict) + self.setup_params = make_parameters("setup.cfg") + self.qgis_plugin_config_params = make_parameters(".qgis-plugin-ci") + self.pyproject_params = make_parameters("pyproject.toml") + self.tx_api_token = os.getenv("tx_api_token") self.github_token = os.getenv("github_token") self.repo = None @@ -59,15 +60,28 @@ def clean_assets(self): print(f" delete {asset.name}") asset.delete_asset() if self.t: - self.t._t.delete_project(self.parameters.project_slug) + self.t._t.delete_project(self.qgis_plugin_config_params.project_slug) + + def test_dict_from_config(self): + with self.subTest(): + self.assertTrue(dict(self.qgis_plugin_config_params)) + self.assertTrue(dict(self.pyproject_params)) + self.assertTrue(dict(self.setup_params)) + + def test_release_from_dot_qgis_plugin_ci(self): + release(self.qgis_plugin_config_params, RELEASE_VERSION_TEST) - def test_release(self): - release(self.parameters, RELEASE_VERSION_TEST) + def test_release_from_pyproject(self): + release(self.pyproject_params, RELEASE_VERSION_TEST) @unittest.skipIf(can_skip_test(), "Missing tx_api_token") def test_release_with_transifex(self): - Translation(self.parameters, tx_api_token=self.tx_api_token) - release(self.parameters, RELEASE_VERSION_TEST, tx_api_token=self.tx_api_token) + Translation(self.qgis_plugin_config_params, tx_api_token=self.tx_api_token) + release( + self.qgis_plugin_config_params, + RELEASE_VERSION_TEST, + tx_api_token=self.tx_api_token, + ) def test_zipname(self): """Tests about the zipname for the QGIS plugin manager. @@ -96,7 +110,7 @@ def test_zipname(self): @unittest.skipIf(can_skip_test(), "Missing github_token") def test_release_upload_github(self): release( - self.parameters, + self.qgis_plugin_config_params, RELEASE_VERSION_TEST, github_token=self.github_token, upload_plugin_repo_github=True, @@ -123,8 +137,8 @@ def test_release_upload_github(self): # compare archive file size gh_release = self.repo.get_release(id=RELEASE_VERSION_TEST) - archive_name = self.parameters.archive_name( - self.parameters.plugin_path, RELEASE_VERSION_TEST + archive_name = self.qgis_plugin_config_params.archive_name( + self.qgis_plugin_config_params.plugin_path, RELEASE_VERSION_TEST ) fs = os.path.getsize(archive_name) print("size: ", fs)