diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 461600fa68b5..6ad79a7549a2 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -4039,6 +4039,16 @@ of "type" specified for this expression block. </xs:restriction> </xs:simpleType> + <xs:simpleType name="RequestMethodType"> + <xs:annotation> + <xs:documentation xml:lang="en">Select a request method, defaults to GET if unspecified</xs:documentation> + </xs:annotation> + <xs:restriction base="xs:string"> + <xs:enumeration value="GET" /> + <xs:enumeration value="POST" /> + </xs:restriction> + </xs:simpleType> + <xs:complexType name="ParamSelectOption"> <xs:annotation> <xs:documentation xml:lang="en"><![CDATA[ @@ -4413,6 +4423,11 @@ used to generate dynamic options. <xs:documentation xml:lang="en">Determine options from data hosted at specified URL. </xs:documentation> </xs:annotation> </xs:attribute> + <xs:attribute name="request_method" type="RequestMethodType"> + <xs:annotation> + <xs:documentation xml:lang="en">Set the request method to use for options provided using from_url. </xs:documentation> + </xs:annotation> + </xs:attribute> <xs:attribute name="from_parameter" type="xs:string"> <xs:annotation> <xs:documentation xml:lang="en">Deprecated.</xs:documentation> @@ -4446,10 +4461,12 @@ used to generate dynamic options. </xs:complexType> <xs:group name="OptionsElement"> <xs:choice> - <xs:element name="filter" type="Filter" minOccurs="0" maxOccurs="unbounded"/> - <xs:element name="column" type="Column" minOccurs="0" maxOccurs="unbounded"/> - <xs:element name="validator" type="Validator" minOccurs="0" maxOccurs="1"/> - <xs:element name="postprocess_expression" type="Expression" minOccurs="0" maxOccurs="1"/> + <xs:element name="filter" type="Filter" minOccurs="0" maxOccurs="unbounded" /> + <xs:element name="column" type="Column" minOccurs="0" maxOccurs="unbounded" /> + <xs:element name="validator" type="Validator" minOccurs="0" maxOccurs="1" /> + <xs:element name="postprocess_expression" type="Expression" minOccurs="0" maxOccurs="1" /> + <xs:element name="request_body" type="RequestBody" minOccurs="0" maxOccurs="1" /> + <xs:element name="request_headers" type="RequestHeaders" minOccurs="0" maxOccurs="1" /> <xs:element name="file" type="xs:string" minOccurs="0" maxOccurs="unbounded"> <xs:annotation> <xs:documentation xml:lang="en">Documentation for file</xs:documentation> @@ -4458,6 +4475,20 @@ used to generate dynamic options. <xs:element name="option" type="ParamDrillDownOption" minOccurs="0" maxOccurs="unbounded"/> </xs:choice> </xs:group> + <xs:complexType name="RequestBody"> + <xs:simpleContent> + <xs:extension base="xs:string"> + <xs:attribute name="type" type="xs:string"/> + </xs:extension> + </xs:simpleContent> + </xs:complexType> + <xs:complexType name="RequestHeaders"> + <xs:simpleContent> + <xs:extension base="xs:string"> + <xs:attribute name="type" type="xs:string"/> + </xs:extension> + </xs:simpleContent> + </xs:complexType> <xs:complexType name="Column"> <xs:annotation> <xs:documentation xml:lang="en"><![CDATA[Optionally contained within an diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py index 28e6fd6f7ae1..5cab45d7fa14 100644 --- a/lib/galaxy/tools/parameters/dynamic_options.py +++ b/lib/galaxy/tools/parameters/dynamic_options.py @@ -4,10 +4,19 @@ """ import copy +import json import logging import os import re +from dataclasses import dataclass from io import StringIO +from typing import ( + Any, + Dict, + Optional, +) + +from typing_extensions import Literal from galaxy.model import ( DatasetCollectionElement, @@ -17,7 +26,10 @@ User, ) from galaxy.tools.expressions import do_eval -from galaxy.util import string_as_bool +from galaxy.util import ( + Element, + string_as_bool, +) from galaxy.util.template import fill_template from . import validation from .cancelable_request import request @@ -538,7 +550,7 @@ def filter_options(self, options, trans, other_values): class DynamicOptions: """Handles dynamically generated SelectToolParameter options""" - def __init__(self, elem, tool_param): + def __init__(self, elem: Element, tool_param): def load_from_parameter(from_parameter, transform_lines=None): obj = self.tool_param for field in from_parameter.split("."): @@ -568,11 +580,7 @@ def load_from_parameter(from_parameter, transform_lines=None): dataset_file = elem.get("from_dataset", None) from_parameter = elem.get("from_parameter", None) self.tool_data_table_name = elem.get("from_data_table", None) - self.from_url = elem.get("from_url") - self.from_url_postprocess = None - from_url_postprocess = elem.find("postprocess_expression") - if from_url_postprocess is not None: - self.from_url_postprocess = from_url_postprocess.text.strip() + self.from_url_options = parse_from_url_options(elem) # Options are defined from a data table loaded by the app self._tool_data_table = None self.elem = elem @@ -781,25 +789,35 @@ def to_triple(values): else: return [str(values[0]), str(values[1]), bool(values[2])] - if self.from_url: + if from_url_options := self.from_url_options: context = User.user_template_environment(trans.user) - url = fill_template(self.from_url, context) + url = fill_template(from_url_options.from_url, context) + request_body = template_or_none(from_url_options.request_body, context) + request_headers = template_or_none(from_url_options.request_headers, context) try: unset_value = object() - cached_value = trans.get_cache_value(url, unset_value) + cached_value = trans.get_cache_value( + (url, from_url_options.request_method, request_body, request_headers), unset_value + ) if cached_value is unset_value: - data = request(url, timeout=10) - trans.set_cache_value(url, data) + data = request( + url=url, + method=from_url_options.request_method, + data=json.loads(request_body) if request_body else None, + headers=json.loads(request_headers) if request_headers else None, + timeout=10, + ) + trans.set_cache_value((url, from_url_options.request_method, request_body, request_headers), data) else: data = cached_value except Exception as e: log.warning("Fetching from url '%s' failed: %s", url, str(e)) data = None - if self.from_url_postprocess: + if from_url_options.postprocess_expression: try: data = do_eval( - self.from_url_postprocess, + from_url_options.postprocess_expression, data, ) except Exception as eval_error: @@ -836,6 +854,44 @@ def column_spec_to_index(self, column_spec): return int(column_spec) +@dataclass +class FromUrlOptions: + from_url: str + request_method: Literal["GET", "POST"] + request_body: Optional[str] + request_headers: Optional[str] + postprocess_expression: Optional[str] + + +def strip_or_none(maybe_string: Optional[str]) -> Optional[str]: + if maybe_string is not None: + return maybe_string.text.strip() + return None + + +def parse_from_url_options(elem: Element) -> Optional[FromUrlOptions]: + from_url = elem.get("from_url") + if from_url: + request_method = elem.get("request_method", "GET") + request_headers = strip_or_none(elem.find("request_headers")) + request_body = strip_or_none(elem.find("request_body")) + postprocess_expression = strip_or_none(elem.find("postprocess_expression")) + return FromUrlOptions( + from_url, + request_method=request_method, + request_headers=request_headers, + request_body=request_body, + postprocess_expression=postprocess_expression, + ) + return None + + +def template_or_none(template: Optional[str], context: Dict[str, Any]) -> Optional[str]: + if template: + return fill_template(template, context=context) + return None + + def _get_ref_data(other_values, ref_name): """ get the list of data sets from ref_name diff --git a/lib/galaxy/work/context.py b/lib/galaxy/work/context.py index a5bb476e0d19..8e4fc74afc21 100644 --- a/lib/galaxy/work/context.py +++ b/lib/galaxy/work/context.py @@ -4,6 +4,7 @@ Dict, List, Optional, + Tuple, ) from typing_extensions import Literal @@ -44,15 +45,15 @@ def __init__( self.__user_current_roles: Optional[List[Role]] = None self.__history = history self._url_builder = url_builder - self._short_term_cache: Dict[str, Any] = {} + self._short_term_cache: Dict[Tuple[str, ...], Any] = {} self.workflow_building_mode = workflow_building_mode self.galaxy_session = galaxy_session - def set_cache_value(self, key: str, value: Any): - self._short_term_cache[key] = value + def set_cache_value(self, args: Tuple[str, ...], value: Any): + self._short_term_cache[args] = value - def get_cache_value(self, key: str, default: Any = None) -> Any: - return self._short_term_cache.get(key, default) + def get_cache_value(self, args: Tuple[str, ...], default: Any = None) -> Any: + return self._short_term_cache.get(args, default) @property def app(self): diff --git a/test/functional/tools/select_from_url.xml b/test/functional/tools/select_from_url.xml index 260b239dc908..60a712e74631 100644 --- a/test/functional/tools/select_from_url.xml +++ b/test/functional/tools/select_from_url.xml @@ -2,7 +2,8 @@ <command><![CDATA[ echo '$url_param_value' > '$param_value' && echo '$url_param_value_postprocessed' > '$param_value_postprocessed' && -echo '$invalid_url_param_value_postprocessed' > '$invalid_param_value_postprocessed' +echo '$invalid_url_param_value_postprocessed' > '$invalid_param_value_postprocessed' && +echo '$url_param_value_header_and_body' > '$param_value_header_and_body' ]]></command> <inputs> <param name="url_param_value" type="select"> @@ -33,17 +34,34 @@ echo '$invalid_url_param_value_postprocessed' > '$invalid_param_value_postproces }]]></postprocess_expression> </options> </param> + <param name="url_param_value_header_and_body" type="select"> + <options from_url="https://postman-echo.com/post" request_method="POST"> + <!-- Example for accessing user secrets via extra preferences --> + <request_headers type="json"> + {"x-api-key": "${__user__.extra_preferences.fake_api_key if $__user__ else "anon"}"} + </request_headers> + <request_body type="json"> + {"name": "value"} + </request_body> + <!-- https://postman-echo.com/post echos values sent to it, so here's we're listing the response headers --> + <postprocess_expression type="ecma5.1"><![CDATA[${ + return Object.keys(inputs.headers).map((header) => [header, header]) + }]]></postprocess_expression> + </options> + </param> </inputs> <outputs> <data format="txt" label="url param value" name="param_value"></data> <data format="txt" label="url param value postprocessed" name="param_value_postprocessed"></data> <data format="txt" label="invalid url param value postprocessed" name="invalid_param_value_postprocessed"></data> + <data format="txt" label="param value for header and body request" name="param_value_header_and_body"></data> </outputs> <tests> <test> <param name="url_param_value" value="dm6" /> <param name="url_param_value_postprocessed" value="chr2L" /> <param name="invalid_url_param_value_postprocessed" value="default" /> + <param name="url_param_value_header_and_body" value="x-api-key" /> <output name="param_value"> <assert_contents> <has_text text="dm6"></has_text> @@ -59,6 +77,11 @@ echo '$invalid_url_param_value_postprocessed' > '$invalid_param_value_postproces <has_text text="default"></has_text> </assert_contents> </output> + <output name="param_value_header_and_body"> + <assert_contents> + <has_text text="x-api-key"></has_text> + </assert_contents> + </output> </test> </tests> </tool> diff --git a/test/unit/app/tools/test_dynamic_options.py b/test/unit/app/tools/test_dynamic_options.py new file mode 100644 index 000000000000..38626d65b991 --- /dev/null +++ b/test/unit/app/tools/test_dynamic_options.py @@ -0,0 +1,57 @@ +from galaxy.app_unittest_utils.galaxy_mock import MockApp +from galaxy.tools.parameters.dynamic_options import DynamicOptions +from galaxy.util import XML +from galaxy.util.bunch import Bunch +from galaxy.work.context import WorkRequestContext + + +def get_from_url_option(): + return DynamicOptions( + XML( + """ +<options from_url="https://usegalaxy.org/api/genomes/dm6" request_method="POST"> + <request_headers type="json"> + {"x-api-key": "${__user__.extra_preferences.resource_api_key if $__user__ else "anon"}"} + </request_headers> + <request_body type="json"> + {"some_key": "some_value"} + </request_body> + <postprocess_expression type="ecma5.1"><![CDATA[${ + if (inputs) { + return Object.values(inputs.chrom_info).map((v) => [v.chrom, v.len]) + } else { + return [["The fallback value", "default"]] + } + }]]></postprocess_expression> +</options> +""" + ), + Bunch(), + ) + + +def test_dynamic_option_parsing(): + from_url_option = get_from_url_option() + assert from_url_option.from_url_options + assert from_url_option.from_url_options.from_url == "https://usegalaxy.org/api/genomes/dm6" + + +def test_dynamic_option_cache(): + app = MockApp() + trans = WorkRequestContext(app=app) + from_url_option = get_from_url_option() + options = from_url_option.from_url_options + assert options + args = (options.from_url, options.request_method, options.request_body, '{"x-api-key": "anon"}') + trans.set_cache_value( + args, + { + "id": "dm6", + "reference": True, + "chrom_info": [{"chrom": "chr2L", "len": 23513712}], + "prev_chroms": False, + "next_chroms": False, + "start_index": 0, + }, + ) + assert from_url_option.get_options(trans, {}) == [["chr2L", "23513712", False]]