Merge pull request #18457 from bwalkowi/add-onedata-templates

Add onedata templates

client/src/api/fileSources.ts

@@ -4,3 +4,4 @@ export type FileSourceTemplateSummary = components["schemas"]["FileSourceTemplat
 export type FileSourceTemplateSummaries = FileSourceTemplateSummary[];
 
 export type UserFileSourceModel = components["schemas"]["UserFileSourceModel"];
+export type FileSourceTypes = UserFileSourceModel["type"];

client/src/api/schema/schema.ts

@@ -8785,7 +8785,7 @@ export interface components {
              * Type
              * @enum {string}
              */
-            type: "ftp" | "posix" | "s3fs" | "azure";
+            type: "ftp" | "posix" | "s3fs" | "azure" | "onedata";
             /** Variables */
             variables?:
                 | (
@@ -13748,7 +13748,7 @@ export interface components {
              * Type
              * @enum {string}
              */
-            type: "aws_s3" | "azure_blob" | "boto3" | "disk" | "generic_s3";
+            type: "aws_s3" | "azure_blob" | "boto3" | "disk" | "generic_s3" | "onedata";
             /** Variables */
             variables?:
                 | (
@@ -16503,7 +16503,7 @@ export interface components {
              * Type
              * @enum {string}
              */
-            type: "aws_s3" | "azure_blob" | "boto3" | "disk" | "generic_s3";
+            type: "aws_s3" | "azure_blob" | "boto3" | "disk" | "generic_s3" | "onedata";
             /**
              * Uuid
              * Format: uuid4
@@ -16578,7 +16578,7 @@ export interface components {
              * Type
              * @enum {string}
              */
-            type: "ftp" | "posix" | "s3fs" | "azure";
+            type: "ftp" | "posix" | "s3fs" | "azure" | "onedata";
             /** Uri Root */
             uri_root: string;
             /**

client/src/components/FileSources/FileSourceTypeSpan.vue

@@ -1,13 +1,18 @@
 <script setup lang="ts">
 import { computed } from "vue";
 
+import type { FileSourceTypes } from "@/api/fileSources";
+
 const MESSAGES = {
     posix: "This is a simple path based storage location that assumes the all the relevant paths are already mounted on the Galaxy server and target worker nodes.",
     s3fs: "This is an remote file source plugin based on the Amazon Simple Storage Service (S3) interface. The AWS interface has become an industry standard and many storage vendors support it and use it to expose 'object' based storage.",
+    azure: "This is an remote file source plugin based on the Azure service.",
+    onedata: "This is an remote file source plugin based on the Onedata service.",
+    ftp: "This is an remote file source plugin based on the FTP protocol.",
 };
 
 interface Props {
-    type: "posix" | "s3fs";
+    type: FileSourceTypes;
 }
 
 const props = defineProps<Props>();

client/src/components/ObjectStore/ObjectStoreTypeSpan.vue

@@ -11,6 +11,7 @@ const MESSAGES = {
     boto3: "This is a storage location based on the Amazon Simple Storage Service (S3) interface, but likely not stored by Amazon. The AWS interface has become an industry standard and many storage vendors support it and use it to expose object based storage.",
     generic_s3:
         "This is a storage location based on the Amazon Simple Storage Service (S3) interface, but likely not stored by Amazon. The AWS interface has become an industry standard and many storage vendors support it and use it to expose object based storage.",
+    onedata: "This is a storage location based on the Onedata system.",
 };
 
 interface Props {

lib/galaxy/config/sample/file_sources_conf.yml.sample

@@ -235,6 +235,6 @@
   id: onedata1
   label: Onedata
   doc: Your Onedata files - configure an access token via user preferences
-  accessToken: ${user.preferences['onedata|access_token']}
-  onezoneDomain: ${user.preferences['onedata|onezone_domain']}
-  disableTlsCertificateValidation: ${user.preferences['onedata|disable_tls_certificate_validation']}
+  access_token: ${user.preferences['onedata|access_token']}
+  onezone_domain: ${user.preferences['onedata|onezone_domain']}
+  disable_tls_certificate_validation: ${user.preferences['onedata|disable_tls_certificate_validation']}

lib/galaxy/config/sample/object_store_conf.sample.yml

@@ -220,21 +220,24 @@ extra_dirs:
 
 type: onedata
 auth:
-  # an access token suitable for data access (allowing calls to the Oneprovider REST API).
+  # An access token suitable for data access (allowing calls to the Oneprovider REST API).
   access_token: ...
 connection:
-  # the domain of the Onezone service (e.g. datahub.egi.eu), or its IP address for
-  # devel instances (see above).
+  # The domain of the Onezone service (e.g. datahub.egi.eu), or its IP address for
+  # devel instances (see above). The minimal supported Onezone version is 21.02.4.
   onezone_domain: datahub.egi.eu
   # Allows connection to Onedata servers that do not present trusted SSL certificates. 
   # SHOULD NOT be used unless you really know what you are doing.
   disable_tls_certificate_validation: false
 space:
-  # the name of the Onedata space where the Galaxy data will be stored.
+  # The name of the Onedata space where the Galaxy data will be stored.
+  # If there is more than one space with the same name, you can explicitly
+  # specify which one to select by using the format <space_name>@<space_id>
+  # (e.g. demo@7285220ecc636075ae5759aec7ad65d3cha8f9).
   name: demo-space
-  # the relative directory path in the space under which the Galaxy data will be stored.
+  # The relative directory path in the space at which the Galaxy data will be stored.
   # Optional, if not provided, the data will be stored in the root of space.
-  path: galaxy-data
+  galaxy_root_dir: galaxy-data
 cache:
   path: database/object_store_cache_s3
   size: 1000

lib/galaxy/config/sample/object_store_conf.xml.sample

@@ -166,24 +166,27 @@
 
     //connection/@onezone_domain -
         the domain of the Onezone service (e.g. datahub.egi.eu), or its IP address for
-        devel instances (see above).
+        devel instances (see above). The minimal supported Onezone version is 21.02.4.
 
     //connection/@disable_tls_certificate_validation - 
         Allows connection to Onedata servers that do not present trusted SSL certificates. 
         SHOULD NOT be used unless you really know what you are doing.
 
     //space/@name -
         the name of the Onedata space where the Galaxy data will be stored.
+        If there is more than one space with the same name, you can explicitly
+        specify which one to select by using the format <space_name>@<space_id>
+        (e.g. demo@7285220ecc636075ae5759aec7ad65d3cha8f9).
 
-    //space/@path -
-        the relative directory path in the space under which the Galaxy data will be stored.
+    //space/@galaxy_root_dir -
+        the relative directory path in the space at which the Galaxy data will be stored.
         Optional, if not provided, the data will be stored in the root of space.
 -->
 <!--
 <object_store type="onedata">
     <auth access_token="..." />
     <connection onezone_domain="datahub.egi.eu" disable_tls_certificate_validation="False"/>
-    <space name="demo-space" path="galaxy-data" />
+    <space name="demo-space" galaxy_root_dir="galaxy-data" />
     <cache path="database/object_store_cache" size="1000" cache_updated_data="True" />
     <extra_dir type="job_work" path="database/job_working_directory_onedata"/>
     <extra_dir type="temp" path="database/tmp_onedata"/>

lib/galaxy/config/sample/user_preferences_extra_conf.yml.sample

@@ -141,7 +141,7 @@ preferences:
         description: Your Onedata account
         inputs:
             - name: onezone_domain
-              label: Domain of the Onezone service (e.g. datahub.egi.eu)
+              label: Domain of the Onezone service (e.g. datahub.egi.eu). The minimal supported Onezone version is 21.02.4.
               type: text
               required: False
             - name: access_token

lib/galaxy/dependencies/conditional-requirements.txt

@@ -24,7 +24,7 @@ fs.googledrivefs # type: googledrive
 fs-gcsfs # type: googlecloudstorage
 # fs-gcsfs doesn't pin google-cloud-storage, and old versions log noisy exceptions and break test discovery
 google-cloud-storage>=2.8.0  # type: googlecloudstorage
-fs.onedatarestfs==21.2.5.1 # type: onedata, depends on onedatafilerestclient
+fs.onedatarestfs==21.2.5.2 # type: onedata, depends on onedatafilerestclient
 fs-basespace # type: basespace
 fs-azureblob # type: azure
 

lib/galaxy/dependencies/dev-requirements.txt

@@ -82,7 +82,7 @@ myst-parser==3.0.1 ; python_version >= "3.8" and python_version < "3.13"
 nh3==0.2.18 ; python_version >= "3.8" and python_version < "3.13"
 numpy==1.24.4 ; python_version >= "3.8" and python_version < "3.9"
 numpy==2.0.0 ; python_version >= "3.9" and python_version < "3.13"
-onedatafilerestclient==21.2.5.1 ; python_version >= "3.8" and python_version < "3.13"
+onedatafilerestclient==21.2.5.2 ; python_version >= "3.8" and python_version < "3.13"
 outcome==1.3.0.post0 ; python_version >= "3.8" and python_version < "3.13"
 packaging==24.1 ; python_version >= "3.8" and python_version < "3.13"
 pathspec==0.12.1 ; python_version >= "3.8" and python_version < "3.13"

lib/galaxy/files/sources/onedata.py

@@ -5,6 +5,7 @@
 
 from typing import Optional
 
+from galaxy.util import mapped_chars
 from . import FilesSourceOptions
 from ._pyfilesystem2 import PyFilesystem2FilesSource
 
@@ -22,11 +23,20 @@ class OnedataFilesSource(PyFilesystem2FilesSource):
 
     def _open_fs(self, user_context=None, opts: Optional[FilesSourceOptions] = None):
         props = self._serialization_props(user_context)
-        onezone_domain = props.pop("onezoneDomain", "") or ""
+
+        access_token = props.pop("access_token", "") or ""
+        onezone_domain = props.pop("onezone_domain", "") or ""
         onezone_domain = remove_prefix("http://", remove_prefix("https://", onezone_domain))
-        access_token = props.pop("accessToken", "") or ""
-        disable_tls_certificate_validation = props.pop("disableTlsCertificateValidation", False) or False
-        handle = OnedataRESTFS(onezone_domain, access_token, verify_ssl=not disable_tls_certificate_validation)
+        disable_tls_certificate_validation = props.pop("disable_tls_certificate_validation", False) or False
+
+        alt_space_fqn_separators = [mapped_chars["@"]] if "@" in mapped_chars else None
+
+        handle = OnedataRESTFS(
+            onezone_domain,
+            access_token,
+            alt_space_fqn_separators=alt_space_fqn_separators,
+            verify_ssl=not disable_tls_certificate_validation,
+        )
         return handle
 
 

lib/galaxy/files/templates/examples/onedata.yml

@@ -0,0 +1,44 @@
+- id: onedata
+  version: 0
+  name: Onedata Storage
+  description: |
+    If you have an account in Onedata, you can use such a file source to upload
+    your data directly from Onedata to Galaxy. You can also perform
+    an "Export datasets to remote files source" of a dataset from Galaxy to
+    Onedata.
+
+    The minimal supported Onezone version is 21.02.4.
+    More information on Onedata can be found on [Onedata's website](https://onedata.org/#/home).
+  variables:
+    onezone_domain:
+      label: Onezone domain
+      type: string
+      help: Domain of the Onezone service (e.g. datahub.egi.eu) to connect to.
+    disable_tls_certificate_validation:
+      label: Disable tls certificate validation?
+      type: boolean
+      help: |
+        Allows connection to Onedata servers that do not present trusted SSL certificates.
+        SHOULD NOT be used unless you really know what you are doing.
+      default: False
+    writable:
+      label: Writable?
+      type: boolean
+      help: |
+        Allow Galaxy to write data to this Onedata storage.
+        Requires an access token with write data access.
+      default: False
+  secrets:
+    access_token:
+      label: Access Token
+      help: |
+        Your [access token](https://onedata.org/#/home/documentation/topic/stable/tokens),
+        suitable for REST API access in a Oneprovider service. May be confined
+        to read-only data access, unless you wish to use the
+        "Export datasets to remote files source" tool (write permissions are needed then).
+  configuration:
+    type: onedata
+    access_token: "{{ secrets.access_token }}"
+    onezone_domain: "{{ variables.onezone_domain }}"
+    disable_tls_certificate_validation: "{{ variables.disable_tls_certificate_validation }}"
+    writable: "{{ variables.writable }}"

lib/galaxy/files/templates/models.py

@@ -25,7 +25,7 @@
     UserDetailsDict,
 )
 
-FileSourceTemplateType = Literal["ftp", "posix", "s3fs", "azure"]
+FileSourceTemplateType = Literal["ftp", "posix", "s3fs", "azure", "onedata"]
 
 
 class PosixFileSourceTemplateConfiguration(StrictModel):
@@ -104,17 +104,37 @@ class AzureFileSourceConfiguration(StrictModel):
     writable: bool = False
 
 
+class OnedataFileSourceTemplateConfiguration(StrictModel):
+    type: Literal["onedata"]
+    access_token: Union[str, TemplateExpansion]
+    onezone_domain: Union[str, TemplateExpansion]
+    disable_tls_certificate_validation: Union[bool, TemplateExpansion] = False
+    writable: Union[bool, TemplateExpansion] = False
+    template_start: Optional[str] = None
+    template_end: Optional[str] = None
+
+
+class OnedataFileSourceConfiguration(StrictModel):
+    type: Literal["onedata"]
+    access_token: str
+    onezone_domain: str
+    disable_tls_certificate_validation: bool = False
+    writable: bool = False
+
+
 FileSourceTemplateConfiguration = Union[
     PosixFileSourceTemplateConfiguration,
     S3FSFileSourceTemplateConfiguration,
     FtpFileSourceTemplateConfiguration,
     AzureFileSourceTemplateConfiguration,
+    OnedataFileSourceTemplateConfiguration,
 ]
 FileSourceConfiguration = Union[
     PosixFileSourceConfiguration,
     S3FSFileSourceConfiguration,
     FtpFileSourceConfiguration,
     AzureFileSourceConfiguration,
+    OnedataFileSourceConfiguration,
 ]
 
 
@@ -175,6 +195,7 @@ def template_to_configuration(
     "posix": PosixFileSourceConfiguration,
     "s3fs": S3FSFileSourceConfiguration,
     "azure": AzureFileSourceConfiguration,
+    "onedata": OnedataFileSourceConfiguration,
 }
 
 

lib/galaxy/objectstore/onedata.py

@@ -15,7 +15,10 @@
 except ImportError:
     OnedataFileRESTClient = None
 
-from galaxy.util import string_as_bool
+from galaxy.util import (
+    mapped_chars,
+    string_as_bool,
+)
 from ._caching_base import CachingConcreteObjectStore
 from .caching import (
     enable_cache_monitor,
@@ -44,7 +47,7 @@ def _parse_config_xml(config_xml):
 
         space_xml = _get_config_xml_elements(config_xml, "space")[0]
         space_name = space_xml.get("name")
-        galaxy_root_dir = space_xml.get("path", "")
+        galaxy_root_dir = space_xml.get("galaxy_root_dir", "")
 
         cache_dict = parse_caching_config_dict_from_xml(config_xml)
 
@@ -107,8 +110,8 @@ def __init__(self, config, config_dict):
 
         cache_dict = config_dict.get("cache") or {}
         self.enable_cache_monitor, self.cache_monitor_interval = enable_cache_monitor(config, config_dict)
-        self.cache_size = cache_dict["size"] or self.config.object_store_cache_size
-        self.staging_path = cache_dict["path"] or self.config.object_store_cache_path
+        self.cache_size = cache_dict.get("size") or self.config.object_store_cache_size
+        self.staging_path = cache_dict.get("path") or self.config.object_store_cache_path
         self.cache_updated_data = cache_dict.get("cache_updated_data", True)
 
         extra_dirs = {e["type"]: e["path"] for e in config_dict.get("extra_dirs", [])}
@@ -125,8 +128,14 @@ def _initialize(self):
             f"(disable_tls_certificate_validation={self.disable_tls_certificate_validation})"
         )
 
+        alt_space_fqn_separators = [mapped_chars["@"]] if "@" in mapped_chars else None
         verify_ssl = not self.disable_tls_certificate_validation
-        self._client = OnedataFileRESTClient(self.onezone_domain, self.access_token, verify_ssl=verify_ssl)
+        self._client = OnedataFileRESTClient(
+            self.onezone_domain,
+            self.access_token,
+            alt_space_fqn_separators=alt_space_fqn_separators,
+            verify_ssl=verify_ssl,
+        )
 
         self._ensure_staging_path_writable()
         self._start_cache_monitor_if_needed()