Adding support for pyproject.toml files

- pyproject.toml are becoming the standard nowadays for packaging Python applications so I felt it would be nice to have. - did some DRY-cleaning, moving the logic initializing the argparse Parameters to a stand-alone top level function. It could be an instance method of Parameters too, it's easy to refactor from one to the other. - added tests leveraging the two aforementioend changes - reduced time complexity when collecting metadata - sprinkled some type hints here and there README --- collecting metadata: O(n) from O(n^2)
opengisch · Jun 10, 2023 · 3920673 · 3920673
1 parent 2760451
commit 3920673
Show file tree

Hide file tree

Showing 7 changed files with 158 additions and 71 deletions.
diff --git a/README.md b/README.md
@@ -53,10 +53,19 @@ commands:
 
 ## Requirements
 
-- The code is under a **git** repository (`git archive` is used to bundle the plugin)
-- There is no uncommitted changes when doing a package/release (there is an option to allow this)
-- A configuration at the top directory either in `.qgis-plugin-ci` or in `setup.cfg` with a `[qgis-plugin-ci]` section.
-- The source files of the plugin are within a sub-directory. The name of this directory will be used for the zip file.
+- The code is under a **git** repository (`git archive` is used to bundle the plugin).
+- There is no uncommitted changes when doing a package/release (althought there is an option to bypass this requirement).
+- A configuration at the top directory either in `.qgis-plugin-ci` or in `setup.cfg` or `pyproject.toml` with a `[qgis-plugin-ci]` section with the following fields:
+    - github_organization_slug (unless you're running from Travis CI)
+    - plugin_path
+    - project_slug (unless you're running from Travis CI)
+- The source files of the plugin are within a sub-directory, with among others, a `metadata.txt` file with the following fields:
+    - description
+    - qgisMinimumVersion
+    - repository
+	- tracker
+
+See `parameters.py` for more parameters and details. Notice that the name of this directory will be used for the zip file.
 
 ## QRC and UI files
 

diff --git a/qgis_plugin_CI_testing/qgissettingmanager b/qgis_plugin_CI_testing/qgissettingmanager
diff --git a/qgispluginci/cli.py b/qgispluginci/cli.py
@@ -1,18 +1,12 @@
 #!/usr/bin/env python3
-
 import argparse
-import configparser
 import logging
-import os
 from importlib.metadata import version
 
-import yaml
-
 from qgispluginci.changelog import ChangelogParser
-from qgispluginci.exceptions import ConfigurationNotFound
-from qgispluginci.parameters import Parameters
 from qgispluginci.release import release
 from qgispluginci.translation import Translation
+from qgispluginci.utils import make_parameters
 
 __version__ = version("qgis-plugin-ci")
 __title__ = "QGISPluginCI"
@@ -169,28 +163,8 @@ def cli():
 
     exit_val = 0
 
-    if os.path.isfile(".qgis-plugin-ci"):
-        # We read the .qgis-plugin-ci file
-        with open(".qgis-plugin-ci", encoding="utf8") as f:
-            arg_dict = yaml.safe_load(f)
-    else:
-        config = configparser.ConfigParser()
-        config.read("setup.cfg")
-        if "qgis-plugin-ci" in config.sections():
-            # We read the setup.cfg file
-            arg_dict = dict(config.items("qgis-plugin-ci"))
-        else:
-            # We don't have either a .qgis-plugin-ci or a setup.cfg
-            if args.command == "changelog":
-                # but for the "changelog" sub command, the config file is not required, we can continue
-                arg_dict = dict()
-            else:
-                raise ConfigurationNotFound(
-                    ".qgis-plugin-ci or setup.cfg with a 'qgis-plugin-ci' section have not been found."
-                )
-
-    parameters = Parameters(arg_dict)
-
+    # Initialize Parameters
+    parameters = make_parameters(args)
     # CHANGELOG
     if args.command == "changelog":
         try:

diff --git a/qgispluginci/parameters.py b/qgispluginci/parameters.py
@@ -14,6 +14,7 @@
 import os
 import re
 import sys
+from typing import Any, Callable, Iterator, Optional, Tuple
 
 # 3rd party
 from slugify import slugify
@@ -99,7 +100,9 @@ class Parameters:
 
     def __init__(self, definition: dict):
         self.plugin_path = definition.get("plugin_path")
-        self.plugin_name = self.__get_from_metadata("name")
+
+        get_metadata = self.collect_metadata()
+        self.plugin_name = get_metadata("name")
         self.plugin_slug = slugify(self.plugin_name)
         self.project_slug = definition.get(
             "project_slug",
@@ -146,22 +149,22 @@ def __init__(self, definition: dict):
             # This tool can be used outside of a QGIS plugin to read a changelog file
             return
 
-        self.author = self.__get_from_metadata("author", "")
-        self.description = self.__get_from_metadata("description")
-        self.qgis_minimum_version = self.__get_from_metadata("qgisMinimumVersion")
-        self.icon = self.__get_from_metadata("icon", "")
-        self.tags = self.__get_from_metadata("tags", "")
-        self.experimental = self.__get_from_metadata("experimental", False)
-        self.deprecated = self.__get_from_metadata("deprecated", False)
-        self.issue_tracker = self.__get_from_metadata("tracker")
-        self.homepage = self.__get_from_metadata("homepage", "")
+        self.author = get_metadata("author", "")
+        self.description = get_metadata("description")
+        self.qgis_minimum_version = get_metadata("qgisMinimumVersion")
+        self.icon = get_metadata("icon", "")
+        self.tags = get_metadata("tags", "")
+        self.experimental = get_metadata("experimental", False)
+        self.deprecated = get_metadata("deprecated", False)
+        self.issue_tracker = get_metadata("tracker")
+        self.homepage = get_metadata("homepage", "")
         if self.homepage == "":
             logger.warning(
                 "Homepage is not given in the metadata. "
                 "It is a mandatory information to publish "
                 "the plugin on the QGIS official repository."
             )
-        self.repository_url = self.__get_from_metadata("repository")
+        self.repository_url = get_metadata("repository")
 
     @staticmethod
     def archive_name(
@@ -178,20 +181,43 @@ def archive_name(
         experimental = "-experimental" if experimental else ""
         return f"{plugin_name}{experimental}.{release_version}.zip"
 
-    def __get_from_metadata(self, key: str, default_value: any = None) -> str:
-        if not self.plugin_path:
-            return ""
-
+    def collect_metadata(self) -> Callable[[str, Optional[Any]], Any]:
+        """
+        Returns a closure capturing a Dict of metadata, allowing to retrieve one
+        value after the other while also iterating over the file once.
+        """
         metadata_file = f"{self.plugin_path}/metadata.txt"
-        with open(metadata_file) as f:
-            for line in f:
-                m = re.match(rf"{key}\s*=\s*(.*)$", line)
-                if m:
-                    return m.group(1)
-        if default_value is None:
-            logger.error(f"Mandatory key is missing in metadata: {key}")
-            sys.exit(1)
-        return default_value
+        metadata = {}
+
+        with open(metadata_file) as fh:
+            for line in fh:
+                split = line.strip().split("=", 1)
+                if len(split) == 2:
+                    metadata[split[0]] = split[1]
+
+        def get_metadata(key: str, default_value: Optional[Any] = None) -> Any:
+            if not self.plugin_path:
+                return ""
+
+            value = metadata.get(key, None)
+            if value:
+                return value
+            elif default_value is not None:
+                return default_value
+            else:
+                logger.error(f"Mandatory key is missing in metadata: {key}")
+                sys.exit(1)
+
+        return get_metadata
+
+    def __iter__(self) -> Iterator[Tuple[str, Any]]:
+        """Allows to represent attributes as dict, list, etc."""
+        for k in vars(self):
+            yield k, self.__getattribute__(k)
+
+    def __str__(self) -> str:
+        """Allows to represent instances as a string."""
+        return str(dict(self))
 
 
 # ############################################################################

diff --git a/qgispluginci/utils.py b/qgispluginci/utils.py
@@ -1,11 +1,17 @@
+import configparser
 import logging
 import os
 import re
 from math import floor
 from math import log as math_log
 from math import pow
-from typing import Union
+from typing import Any, Dict, Optional, Union
 
+import toml
+import yaml
+
+from qgispluginci.exceptions import ConfigurationNotFound
+from qgispluginci.parameters import Parameters
 from qgispluginci.version_note import VersionNote
 
 # GLOBALS
@@ -29,6 +35,63 @@ def configure_file(source_file: str, dest_file: str, replace: dict):
         f.write(content)
 
 
+def make_parameters(args=None, config_file: Optional[str] = None) -> Parameters:
+    """
+    Make a Dict from a config file or by exploring the filesystem
+    Accepts an argparse Namespace for backward compatibility.
+    """
+    configuration_not_found = ConfigurationNotFound(
+        ".qgis-plugin-ci or setup.cfg or pyproject.toml with a 'qgis-plugin-ci' section have not been found."
+    )
+
+    def explore_config() -> Dict[str, Any]:
+        if os.path.isfile(".qgis-plugin-ci"):
+            # We read the .qgis-plugin-ci file
+            with open(".qgis-plugin-ci", encoding="utf8") as f:
+                arg_dict = yaml.safe_load(f)
+        elif os.path.isfile("pyproject.toml"):
+            # We read the pyproject.toml file
+            with open("pyproject.toml", encoding="utf8") as f:
+                arg_dict = toml.load(f)
+        else:
+            config = configparser.ConfigParser()
+            config.read("setup.cfg")
+            if "qgis-plugin-ci" in config.sections():
+                # We read the setup.cfg file
+                arg_dict = dict(config.items("qgis-plugin-ci"))
+            else:
+                # We don't have either a .qgis-plugin-ci or a setup.cfg
+                if args and args.command == "changelog":
+                    # but for the "changelog" sub command, the config file is not required, we can continue
+                    arg_dict = dict()
+                else:
+                    raise configuration_not_found
+        return arg_dict
+
+    def load_config(filename: str) -> Dict[str, Any]:
+        if filename == "setup.cfg":
+            config = configparser.ConfigParser()
+            config.read(filename)
+            return dict(config.items("qgis-plugin-ci"))
+
+        _, suffix = filename.rsplit(".", 1)
+
+        with open(filename) as f:
+            if suffix == "toml":
+                return toml.load(f)
+            elif suffix in {"yaml", "yml"}:
+                return yaml.safe_load(f)
+
+        raise configuration_not_found
+
+    if config_file:
+        config_dict = load_config(config_file)
+    else:
+        config_dict = explore_config()
+
+    return Parameters(config_dict)
+
+
 def convert_octets(octets: int) -> str:
     """Convert a mount of octets in readable size.
 

diff --git a/requirements/base.txt b/requirements/base.txt
@@ -8,3 +8,4 @@ pyqt5ac>=1.2,<1.3
 python-slugify>=4.0,<8.1
 pyyaml>=5.4,<6.1
 transifex-python>=3.2,<4.0
+toml>=0.10.2
diff --git a/test/test_release.py b/test/test_release.py
@@ -20,7 +20,7 @@
 from qgispluginci.parameters import DASH_WARNING, Parameters
 from qgispluginci.release import release
 from qgispluginci.translation import Translation
-from qgispluginci.utils import replace_in_file
+from qgispluginci.utils import make_parameters, replace_in_file
 
 # Tests
 from .utils import can_skip_test
@@ -31,9 +31,10 @@
 
 class TestRelease(unittest.TestCase):
     def setUp(self):
-        with open(".qgis-plugin-ci", encoding="utf8") as f:
-            arg_dict = yaml.safe_load(f)
-        self.parameters = Parameters(arg_dict)
+        self.setup_params = make_parameters("setup.cfg")
+        self.qgis_plugin_config_params = make_parameters(".qgis-plugin-ci")
+        self.pyproject_params = make_parameters("pyproject.toml")
+
         self.tx_api_token = os.getenv("tx_api_token")
         self.github_token = os.getenv("github_token")
         self.repo = None
@@ -59,15 +60,28 @@ def clean_assets(self):
                     print(f"  delete {asset.name}")
                     asset.delete_asset()
         if self.t:
-            self.t._t.delete_project(self.parameters.project_slug)
+            self.t._t.delete_project(self.qgis_plugin_config_params.project_slug)
+
+    def test_dict_from_config(self):
+        with self.subTest():
+            self.assertTrue(dict(self.qgis_plugin_config_params))
+            self.assertTrue(dict(self.pyproject_params))
+            self.assertTrue(dict(self.setup_params))
+
+    def test_release_from_dot_qgis_plugin_ci(self):
+        release(self.qgis_plugin_config_params, RELEASE_VERSION_TEST)
 
-    def test_release(self):
-        release(self.parameters, RELEASE_VERSION_TEST)
+    def test_release_from_pyproject(self):
+        release(self.pyproject_params, RELEASE_VERSION_TEST)
 
     @unittest.skipIf(can_skip_test(), "Missing tx_api_token")
     def test_release_with_transifex(self):
-        Translation(self.parameters, tx_api_token=self.tx_api_token)
-        release(self.parameters, RELEASE_VERSION_TEST, tx_api_token=self.tx_api_token)
+        Translation(self.qgis_plugin_config_params, tx_api_token=self.tx_api_token)
+        release(
+            self.qgis_plugin_config_params,
+            RELEASE_VERSION_TEST,
+            tx_api_token=self.tx_api_token,
+        )
 
     def test_zipname(self):
         """Tests about the zipname for the QGIS plugin manager.
@@ -96,7 +110,7 @@ def test_zipname(self):
     @unittest.skipIf(can_skip_test(), "Missing github_token")
     def test_release_upload_github(self):
         release(
-            self.parameters,
+            self.qgis_plugin_config_params,
             RELEASE_VERSION_TEST,
             github_token=self.github_token,
             upload_plugin_repo_github=True,
@@ -123,8 +137,8 @@ def test_release_upload_github(self):
 
         # compare archive file size
         gh_release = self.repo.get_release(id=RELEASE_VERSION_TEST)
-        archive_name = self.parameters.archive_name(
-            self.parameters.plugin_path, RELEASE_VERSION_TEST
+        archive_name = self.qgis_plugin_config_params.archive_name(
+            self.qgis_plugin_config_params.plugin_path, RELEASE_VERSION_TEST
         )
         fs = os.path.getsize(archive_name)
         print("size: ", fs)