Merge pull request #19331 from nsoranzo/resource_path_fixes

Use ``resource_path()`` to access datatypes_conf.xml.sample as a package resource

lib/galaxy/authnz/managers.py

@@ -26,7 +26,7 @@
     PSAAuthnz,
 )
 
-OIDC_BACKEND_SCHEMA = resource_path(__package__, "xsd/oidc_backends_config.xsd")
+OIDC_BACKEND_SCHEMA = resource_path(__name__, "xsd/oidc_backends_config.xsd")
 
 log = logging.getLogger(__name__)
 

lib/galaxy/config/__init__.py

@@ -70,7 +70,7 @@
 ISO_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S"
 
 GALAXY_APP_NAME = "galaxy"
-GALAXY_SCHEMAS_PATH = resource_path(__package__, "schemas")
+GALAXY_SCHEMAS_PATH = resource_path(__name__, "schemas")
 GALAXY_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / "config_schema.yml"
 REPORTS_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / "reports_config_schema.yml"
 TOOL_SHED_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / "tool_shed_config_schema.yml"
@@ -512,10 +512,10 @@ def resolve(key):
             if not parent:  # base case: nothing else needs resolving
                 return path
             parent_path = resolve(parent)  # recursively resolve parent path
-            if path is not None:
+            if path:
                 path = os.path.join(parent_path, path)  # resolve path
             else:
-                path = parent_path  # or use parent path
+                log.warning("Trying to resolve path for the '%s' option but it's empty/None", key)
 
             setattr(self, key, path)  # update property
             _cache[key] = path  # cache it!
@@ -539,7 +539,7 @@ def resolve(key):
             if self.is_set(key) and self.paths_to_check_against_root and key in self.paths_to_check_against_root:
                 self._check_against_root(key)
 
-    def _check_against_root(self, key):
+    def _check_against_root(self, key: str):
         def get_path(current_path, initial_path):
             # if path does not exist and was set as relative:
             if not self._path_exists(current_path) and not os.path.isabs(initial_path):
@@ -558,6 +558,8 @@ def get_path(current_path, initial_path):
             return current_path
 
         current_value = getattr(self, key)  # resolved path or list of resolved paths
+        if not current_value:
+            return
         if isinstance(current_value, list):
             initial_paths = listify(self._raw_config[key], do_strip=True)  # initial unresolved paths
             updated_paths = []

lib/galaxy/config/templates.py

@@ -33,7 +33,7 @@ def _get_template_body(template: str) -> str:
 
 def _get_template_path(relpath: str, custom_templates_dir: str) -> Traversable:
     """Return template file path."""
-    default_path = resource_path("galaxy.config", "templates") / relpath
+    default_path = resource_path(__name__, "templates") / relpath
     custom_path = Path(custom_templates_dir) / relpath
     if custom_path.exists():
         return custom_path

lib/galaxy/exceptions/error_codes.py

@@ -43,7 +43,7 @@ def _from_dict(entry):
     return (name, ErrorCode(code, message))
 
 
-error_codes_json = resource_string(__package__, "error_codes.json")
+error_codes_json = resource_string(__name__, "error_codes.json")
 error_codes_by_name: Dict[str, ErrorCode] = {}
 error_codes_by_int_code: Dict[int, ErrorCode] = {}
 

lib/galaxy/files/templates/examples/__init__.py

@@ -2,4 +2,4 @@
 
 
 def get_example(filename) -> str:
-    return resource_string("galaxy.files.templates.examples", filename)
+    return resource_string(__name__, filename)

lib/galaxy/jobs/__init__.py

@@ -387,6 +387,8 @@ def __init__(self, app: MinimalManagerApp):
                             f" release of Galaxy. Please convert to YAML at {self.app.config.job_config_file} or"
                             f" explicitly set `job_config_file` to {job_config_file} to remove this message"
                         )
+                if not job_config_file:
+                    raise OSError()
                 if ".xml" in job_config_file:
                     tree = load(job_config_file)
                     job_config_dict = self.__parse_job_conf_xml(tree)

lib/galaxy/managers/licenses.py

@@ -65,7 +65,7 @@ class LicenseMetadataModel(BaseModel):
     "MPL-2.0",
     "PDDL-1.0",
 ]
-SPDX_LICENSES_STRING = resource_string(__package__, "licenses.json")
+SPDX_LICENSES_STRING = resource_string(__name__, "licenses.json")
 SPDX_LICENSES = json.loads(SPDX_LICENSES_STRING)
 for license in SPDX_LICENSES["licenses"]:
     license["recommended"] = license["licenseId"] in RECOMMENDED_LICENSES

lib/galaxy/managers/markdown_util.py

@@ -767,7 +767,7 @@ def to_pdf_raw(basic_markdown: str, css_paths: Optional[List[str]] = None) -> by
         output_file.write(as_html)
         output_file.close()
         html = weasyprint.HTML(filename=index)
-        stylesheets = [weasyprint.CSS(string=resource_string(__package__, "markdown_export_base.css"))]
+        stylesheets = [weasyprint.CSS(string=resource_string(__name__, "markdown_export_base.css"))]
         for css_path in css_paths:
             with open(css_path) as f:
                 css_content = f.read()

lib/galaxy/navigation/data.py

@@ -5,6 +5,6 @@
 
 
 def load_root_component() -> Component:
-    new_data_yaml = resource_string(__package__, "navigation.yml")
+    new_data_yaml = resource_string(__name__, "navigation.yml")
     navigation_raw = yaml.safe_load(new_data_yaml)
     return Component.from_dict("root", navigation_raw)

lib/galaxy/objectstore/examples/__init__.py

@@ -2,4 +2,4 @@
 
 
 def get_example(filename: str) -> str:
-    return resource_string("galaxy.objectstore.examples", filename)
+    return resource_string(__name__, filename)

lib/galaxy/objectstore/templates/examples/__init__.py

@@ -2,4 +2,4 @@
 
 
 def get_example(filename) -> str:
-    return resource_string("galaxy.objectstore.templates.examples", filename)
+    return resource_string(__name__, filename)

lib/galaxy/tool_util/client/landing.py

@@ -24,7 +24,7 @@
 
 
 def load_default_catalog():
-    catalog_yaml = resource_string("galaxy.tool_util.client", "landing_catalog.sample.yml")
+    catalog_yaml = resource_string(__name__, "landing_catalog.sample.yml")
     return yaml.safe_load(catalog_yaml)
 
 

lib/galaxy/tool_util/linters/datatypes.py

@@ -2,6 +2,7 @@
 from typing import (
     Set,
     TYPE_CHECKING,
+    Union,
 )
 
 # from galaxy import config
@@ -10,15 +11,17 @@
     listify,
     parse_xml,
 )
+from galaxy.util.resources import resource_path
 
 if TYPE_CHECKING:
     from galaxy.tool_util.lint import LintContext
     from galaxy.tool_util.parser import ToolSource
+    from galaxy.util.resources import Traversable
 
-DATATYPES_CONF = os.path.join(os.path.dirname(__file__), "datatypes_conf.xml.sample")
+DATATYPES_CONF = resource_path(__name__, "datatypes_conf.xml.sample")
 
 
-def _parse_datatypes(datatype_conf_path: str) -> Set[str]:
+def _parse_datatypes(datatype_conf_path: Union[str, "Traversable"]) -> Set[str]:
     datatypes = set()
     tree = parse_xml(datatype_conf_path)
     root = tree.getroot()

lib/galaxy/tool_util/ontologies/ontology_data.py

@@ -27,7 +27,7 @@ def _multi_dict_mapping(content: str) -> Dict[str, List[str]]:
 
 
 def _read_ontology_data_text(filename: str) -> str:
-    return resource_string(__package__, filename)
+    return resource_string(__name__, filename)
 
 
 BIOTOOLS_MAPPING_FILENAME = "biotools_mappings.tsv"

lib/galaxy/tool_util/upgrade/__init__.py

@@ -46,7 +46,7 @@ class AdviceCode(TypedDict):
     url: NotRequired[str]
 
 
-upgrade_codes_json = resource_string(__package__, "upgrade_codes.json")
+upgrade_codes_json = resource_string(__name__, "upgrade_codes.json")
 upgrade_codes_by_name: Dict[str, AdviceCode] = {}
 
 for name, upgrade_object in loads(upgrade_codes_json).items():

lib/galaxy/util/__init__.py

@@ -45,6 +45,7 @@
     Optional,
     overload,
     Tuple,
+    TYPE_CHECKING,
     TypeVar,
     Union,
 )
@@ -75,6 +76,7 @@
 LXML_AVAILABLE = True
 try:
     from lxml import etree
+    from lxml.etree import DocumentInvalid
 
     # lxml.etree.Element is a function that returns a new instance of the
     # lxml.etree._Element class. This class doesn't have a proper __init__()
@@ -123,6 +125,10 @@ def XML(text: Union[str, bytes]) -> Element:
         XML,
     )
 
+    class DocumentInvalid(Exception):  # type: ignore[no-redef]
+        pass
+
+
 from . import requests
 from .custom_logging import get_logger
 from .inflection import Inflector
@@ -142,6 +148,9 @@ def shlex_join(split_command):
         return " ".join(map(shlex.quote, split_command))
 
 
+if TYPE_CHECKING:
+    from galaxy.util.resources import Traversable
+
 inflector = Inflector()
 
 log = get_logger(__name__)
@@ -333,7 +342,10 @@ def unique_id(KEY_SIZE=128):
 
 
 def parse_xml(
-    fname: StrPath, strip_whitespace=True, remove_comments=True, schemafname: Union[StrPath, None] = None
+    fname: Union[StrPath, "Traversable"],
+    strip_whitespace: bool = True,
+    remove_comments: bool = True,
+    schemafname: Union[StrPath, None] = None,
 ) -> ElementTree:
     """Returns a parsed xml tree"""
     parser = None
@@ -348,8 +360,10 @@ def parse_xml(
             schema_root = etree.XML(schema_file.read())
             schema = etree.XMLSchema(schema_root)
 
+    source = Path(fname) if isinstance(fname, (str, os.PathLike)) else fname
     try:
-        tree = cast(ElementTree, etree.parse(str(fname), parser=parser))
+        with source.open("rb") as f:
+            tree = cast(ElementTree, etree.parse(f, parser=parser))
         root = tree.getroot()
         if strip_whitespace:
             for elem in root.iter("*"):
@@ -359,15 +373,10 @@ def parse_xml(
                     elem.tail = elem.tail.strip()
         if schema:
             schema.assertValid(tree)
-    except OSError as e:
-        if e.errno is None and not os.path.exists(fname):  # type: ignore[unreachable]
-            # lxml doesn't set errno
-            e.errno = errno.ENOENT  # type: ignore[unreachable]
-        raise
     except etree.ParseError:
         log.exception("Error parsing file %s", fname)
         raise
-    except etree.DocumentInvalid:
+    except DocumentInvalid:
         log.exception("Validation of file %s failed", fname)
         raise
     return tree

lib/galaxy/util/resources.py

@@ -3,17 +3,17 @@
 
 import sys
 
-if sys.version_info >= (3, 9):
+if sys.version_info >= (3, 12):
     from importlib.resources import (
         as_file,
         files,
-        Package as Anchor,
     )
+    from importlib.resources.abc import Traversable
 
-    if sys.version_info >= (3, 12):
-        from importlib.resources.abc import Traversable
+    if sys.version_info >= (3, 13):
+        from importlib.resources import Anchor
     else:
-        from importlib.abc import Traversable
+        from importlib.resources import Package as Anchor
 else:
     from importlib_resources import (
         as_file,
@@ -23,20 +23,24 @@
     from importlib_resources.abc import Traversable
 
 
-def resource_path(package_or_requirement: Anchor, resource_name: str) -> Traversable:
+def resource_path(anchor: Anchor, resource_name: str) -> Traversable:
     """
     Return specified resource as a Traversable.
+
+    anchor is either a module object or a module name as a string.
     """
-    return files(package_or_requirement).joinpath(resource_name)
+    return files(anchor).joinpath(resource_name)
 
 
-def resource_string(package_or_requirement: Anchor, resource_name: str) -> str:
+def resource_string(anchor: Anchor, resource_name: str) -> str:
     """
     Return specified resource as a string.
 
     Replacement function for pkg_resources.resource_string, but returns unicode string instead of bytestring.
+
+    anchor is either a module object or a module name as a string.
     """
-    return resource_path(package_or_requirement, resource_name).read_text()
+    return resource_path(anchor, resource_name).read_text()
 
 
 __all__ = (

lib/galaxy/util/rules_dsl.py

@@ -12,7 +12,7 @@
 
 
 def get_rules_specification():
-    return yaml.safe_load(resource_string(__package__, "rules_dsl_spec.yml"))
+    return yaml.safe_load(resource_string(__name__, "rules_dsl_spec.yml"))
 
 
 def _ensure_rule_contains_keys(rule, keys):

lib/galaxy/web/framework/base.py

@@ -35,7 +35,7 @@
 #: time of the most recent server startup
 server_starttime = int(time.time())
 try:
-    meta_json = json.loads(resource_string(__package__, "meta.json"))
+    meta_json = json.loads(resource_string(__name__, "meta.json"))
     server_starttime = meta_json.get("epoch") or server_starttime
 except Exception:
     meta_json = {}

lib/galaxy/webapps/base/webapp.py

@@ -181,7 +181,7 @@ def build_apispec(self):
     def create_mako_template_lookup(self, galaxy_app, name):
         paths = []
         base_package = (
-            "tool_shed.webapp" if galaxy_app.name == "tool_shed" else "galaxy.webapps.base"
+            "tool_shed.webapp" if galaxy_app.name == "tool_shed" else __name__
         )  # reports has templates in galaxy package
         base_template_path = resource_path(base_package, "templates")
         with ExitStack() as stack:

lib/galaxy_test/base/populators.py

@@ -136,10 +136,10 @@
 CWL_TOOL_DIRECTORY = os.path.join(galaxy_root_path, "test", "functional", "tools", "cwl_tools")
 
 # Simple workflow that takes an input and call cat wrapper on it.
-workflow_str = resource_string(__package__, "data/test_workflow_1.ga")
+workflow_str = resource_string(__name__, "data/test_workflow_1.ga")
 # Simple workflow that takes an input and filters with random lines twice in a
 # row - first grabbing 8 lines at random and then 6.
-workflow_random_x2_str = resource_string(__package__, "data/test_workflow_2.ga")
+workflow_random_x2_str = resource_string(__name__, "data/test_workflow_2.ga")
 
 DEFAULT_TIMEOUT = 60  # Secs to wait for state to turn ok
 
@@ -1821,7 +1821,7 @@ def load_random_x2_workflow(self, name: str) -> dict:
     def load_workflow_from_resource(self, name: str, filename: Optional[str] = None) -> dict:
         if filename is None:
             filename = f"data/{name}.ga"
-        content = resource_string(__package__, filename)
+        content = resource_string(__name__, filename)
         return self.load_workflow(name, content=content)
 
     def simple_workflow(self, name: str, **create_kwds) -> str:

lib/tool_shed/test/base/populators.py

@@ -53,7 +53,7 @@
 HasRepositoryId = Union[str, Repository]
 
 DEFAULT_PREFIX = "repofortest"
-TEST_DATA_REPO_FILES = resource_path(__package__, "../test_data")
+TEST_DATA_REPO_FILES = resource_path(__name__, "../test_data")
 COLUMN_MAKER_PATH = TEST_DATA_REPO_FILES.joinpath("column_maker/column_maker.tar")
 COLUMN_MAKER_1_1_1_PATH = TEST_DATA_REPO_FILES.joinpath("column_maker/column_maker_1.1.1.tar")
 DEFAULT_COMMIT_MESSAGE = "a test commit message"

lib/tool_shed/test/functional/test_shed_repositories.py

@@ -20,7 +20,7 @@
     skip_if_api_v2,
 )
 
-COLUMN_MAKER_PATH = resource_path(__package__, "../test_data/column_maker/column_maker.tar")
+COLUMN_MAKER_PATH = resource_path(__name__, "../test_data/column_maker/column_maker.tar")
 
 
 # test_0000 tests commit_message  - find a way to test it here

packages/util/setup.cfg

@@ -36,7 +36,7 @@ install_requires =
     bleach
     boltons
     docutils!=0.17,!=0.17.1
-    importlib-resources;python_version<'3.9'
+    importlib-resources>=5.10.0;python_version<'3.12'
     packaging
     pyparsing
     PyYAML

packages/web_apps/setup.cfg

@@ -46,7 +46,6 @@ install_requires =
     fastapi>=0.101.0
     gunicorn
     gxformat2
-    importlib-resources;python_version<'3.9'
     Mako
     MarkupSafe
     Paste