diff --git a/client/src/components/Page/PageList.test.js b/client/src/components/Page/PageList.test.js index a72af4eda41c..d86af44fa3e8 100644 --- a/client/src/components/Page/PageList.test.js +++ b/client/src/components/Page/PageList.test.js @@ -31,6 +31,8 @@ describe("PgeList.vue", () => { offset: 0, sort_by: "update_time", sort_desc: true, + show_published: false, + show_shared: true, }; const publishedGridApiParams = { ...personalGridApiParams, diff --git a/client/src/components/Page/PageList.vue b/client/src/components/Page/PageList.vue index ef446e41ea67..e0d3d8ebaca5 100644 --- a/client/src/components/Page/PageList.vue +++ b/client/src/components/Page/PageList.vue @@ -177,7 +177,11 @@ export default { }, computed: { dataProviderParameters() { - const extraParams = { search: this.effectiveFilter }; + const extraParams = { + search: this.effectiveFilter, + show_published: false, + show_shared: true, + }; if (this.published) { extraParams.show_published = true; extraParams.show_shared = false; diff --git a/client/src/schema/schema.ts b/client/src/schema/schema.ts index 684a6e6459d8..789deebb50c3 100644 --- a/client/src/schema/schema.ts +++ b/client/src/schema/schema.ts @@ -13546,7 +13546,7 @@ export interface operations { header?: { "run-as"?: string; }; - /** @description The ID of the History. */ + /** @description History ID or any string. */ /** @description The ID of the item (`HDA`/`HDCA`) contained in the history. */ /** * @description The type of the target history element. diff --git a/config/plugins/visualizations/jqplot/jqplot_bar/src/script.js b/config/plugins/visualizations/jqplot/jqplot_bar/src/script.js index bec0b8b510a8..89063ee29a35 100644 --- a/config/plugins/visualizations/jqplot/jqplot_bar/src/script.js +++ b/config/plugins/visualizations/jqplot/jqplot_bar/src/script.js @@ -261,7 +261,7 @@ window.bundleEntries.jqplot_box = function (options) { chart : options.chart, dataset_id : dataset.id, dataset_groups : dataset_groups, - targets : options.targets, + target : options.target, makeConfig : function( groups, plot_config ){ var boundary = getDomains( groups, 'x' ); $.extend( true, plot_config, { diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index d16592821cde..351cfc2c3edf 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -91,6 +91,7 @@ ) from galaxy.queue_worker import ( GalaxyQueueWorker, + reload_toolbox, send_local_control_task, ) from galaxy.quota import ( @@ -773,6 +774,10 @@ def __init__(self, **kwargs) -> None: # Start web stack message handling self.application_stack.register_postfork_function(self.application_stack.start) self.application_stack.register_postfork_function(self.queue_worker.bind_and_start) + # Reload toolbox to pick up changes to toolbox made after master was ready + self.application_stack.register_postfork_function( + lambda: reload_toolbox(self, save_integrated_tool_panel=False), post_fork_only=True + ) # Delay toolbox index until after startup self.application_stack.register_postfork_function( lambda: send_local_control_task(self, "rebuild_toolbox_search_index") diff --git a/lib/galaxy/authnz/psa_authnz.py b/lib/galaxy/authnz/psa_authnz.py index cc571dc65220..0dc084727980 100644 --- a/lib/galaxy/authnz/psa_authnz.py +++ b/lib/galaxy/authnz/psa_authnz.py @@ -30,6 +30,8 @@ from galaxy.util import DEFAULT_SOCKET_TIMEOUT from . import IdentityProvider +log = logging.getLogger(__name__) + # key: a component name which PSA requests. # value: is the name of a class associated with that key. DEFAULTS = {"STRATEGY": "Strategy", "STORAGE": "Storage"} @@ -170,9 +172,14 @@ def refresh(self, trans, user_authnz_token): if not user_authnz_token or not user_authnz_token.extra_data: return False # refresh tokens if they reached their half lifetime - if int(user_authnz_token.extra_data["auth_time"]) + int(user_authnz_token.extra_data["expires"]) / 2 <= int( - time.time() - ): + if "expires" in user_authnz_token.extra_data: + expires = user_authnz_token.extra_data["expires"] + elif "expires_in" in user_authnz_token.extra_data: + expires = user_authnz_token.extra_data["expires_in"] + else: + log.debug("No `expires` or `expires_in` key found in token extra data, cannot refresh") + return False + if int(user_authnz_token.extra_data["auth_time"]) + int(expires) / 2 <= int(time.time()): on_the_fly_config(trans.sa_session) if self.config["provider"] == "azure": self.refresh_azure(user_authnz_token) diff --git a/lib/galaxy/managers/context.py b/lib/galaxy/managers/context.py index 2a19c85ef533..a36ceecb396a 100644 --- a/lib/galaxy/managers/context.py +++ b/lib/galaxy/managers/context.py @@ -159,17 +159,6 @@ def sa_session(self) -> galaxy_scoped_session: """ return self.app.model.session - def expunge_all(self): - """Expunge all the objects in Galaxy's SQLAlchemy sessions.""" - app = self.app - context = app.model.context - context.expunge_all() - # This is a bit hacky, should refctor this. Maybe refactor to app -> expunge_all() - if hasattr(app, "install_model"): - install_model = app.install_model - if install_model != app.model: - install_model.context.expunge_all() - def get_toolbox(self): """Returns the application toolbox. diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 48f7c9eeaa50..7ae306105b4b 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -10746,6 +10746,20 @@ def __repr__(self): # .preferences[pref_name] = pref_value User.preferences = association_proxy("_preferences", "value", creator=UserPreference) +# Optimized version of getting the current Galaxy session. +# See https://github.com/sqlalchemy/sqlalchemy/discussions/7638 for approach +session_partition = select( + GalaxySession, + func.row_number().over(order_by=GalaxySession.update_time, partition_by=GalaxySession.user_id).label("index"), +).alias() +partitioned_session = aliased(GalaxySession, session_partition) +User.current_galaxy_session = relationship( + partitioned_session, + primaryjoin=and_(partitioned_session.user_id == User.id, session_partition.c.index < 2), + uselist=False, + viewonly=True, +) + @event.listens_for(HistoryDatasetCollectionAssociation, "init") def receive_init(target, args, kwargs): diff --git a/lib/galaxy/queue_worker.py b/lib/galaxy/queue_worker.py index 6bc02aa4005c..af28f0655cbb 100644 --- a/lib/galaxy/queue_worker.py +++ b/lib/galaxy/queue_worker.py @@ -238,7 +238,7 @@ def reload_tool_data_tables(app, **kwargs): def rebuild_toolbox_search_index(app, **kwargs): - if app.is_webapp: + if app.is_webapp and app.database_heartbeat.is_config_watcher: if app.toolbox_search.index_count < app.toolbox._reload_count: app.reindex_tool_search() else: diff --git a/lib/galaxy/schema/invocation.py b/lib/galaxy/schema/invocation.py index fa0e741966ef..0d2617a25862 100644 --- a/lib/galaxy/schema/invocation.py +++ b/lib/galaxy/schema/invocation.py @@ -53,7 +53,10 @@ def get(self, key: Any, default: Any = None) -> Any: if key == "workflow_step_id": return self._obj.workflow_step.order_index elif key == "dependent_workflow_step_id": - return self._obj.dependent_workflow_step.order_index + if self._obj.dependent_workflow_step_id: + return self._obj.dependent_workflow_step.order_index + else: + return default return super().get(key, default) diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index e468a14ad67d..60c6b935a251 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -923,6 +923,9 @@ def __parse_param_elem(param_elem, i=0): else: value = None + if value is None and attrib.get("location", None) is not None: + value = os.path.basename(attrib["location"]) + children_elem = param_elem if children_elem is not None: # At this time, we can assume having children only diff --git a/lib/galaxy/tools/cache.py b/lib/galaxy/tools/cache.py index 3704172198f7..00fcb610518a 100644 --- a/lib/galaxy/tools/cache.py +++ b/lib/galaxy/tools/cache.py @@ -6,6 +6,10 @@ import tempfile import zlib from threading import Lock +from typing import ( + Dict, + Optional, +) from sqlitedict import SqliteDict @@ -134,7 +138,7 @@ class ToolCache: def __init__(self): self._lock = Lock() - self._hash_by_tool_paths = {} + self._hash_by_tool_paths: Dict[str, ToolHash] = {} self._tools_by_path = {} self._tool_paths_by_id = {} self._macro_paths_by_id = {} @@ -195,9 +199,12 @@ def _should_cleanup(self, config_filename): try: new_mtime = os.path.getmtime(config_filename) tool_hash = self._hash_by_tool_paths.get(config_filename) - if tool_hash.modtime < new_mtime: - if md5_hash_file(config_filename) != tool_hash.hash: + if tool_hash and tool_hash.modtime_less_than(new_mtime): + if not tool_hash.hash_equals(md5_hash_file(config_filename)): return True + else: + # No change of content, so not necessary to calculate the md5 checksum every time + tool_hash.modtime = new_mtime tool = self._tools_by_path[config_filename] for macro_path in tool._macro_paths: new_mtime = os.path.getmtime(macro_path) @@ -256,13 +263,37 @@ def reset_status(self): class ToolHash: - def __init__(self, path, modtime=None, lazy_hash=False): + def __init__(self, path: str, modtime: Optional[float] = None, lazy_hash: bool = False): self.path = path - self.modtime = modtime or os.path.getmtime(path) + self._modtime = modtime or os.path.getmtime(path) self._tool_hash = None if not lazy_hash: self.hash # noqa: B018 + def modtime_less_than(self, other_modtime: float): + if self._modtime is None: + # For the purposes of the tool cache, + # if we haven't seen the modtime we consider it not equal + return True + return self._modtime < other_modtime + + def hash_equals(self, other_hash: Optional[str]): + if self._tool_hash is None or other_hash is None: + # For the purposes of the tool cache, + # if we haven't seen the hash yet we consider it not equal + return False + return self.hash == other_hash + + @property + def modtime(self) -> float: + if self._modtime is None: + self._modtime = os.path.getmtime(self.path) + return self._modtime + + @modtime.setter + def modtime(self, new_value: float): + self._modtime = new_value + @property def hash(self): if self._tool_hash is None: diff --git a/lib/galaxy/tools/test.py b/lib/galaxy/tools/test.py index b097483ca4a7..ea84995943aa 100644 --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -184,16 +184,25 @@ def _process_raw_inputs( param_value = raw_input_dict["value"] param_extra = raw_input_dict["attributes"] location = param_extra.get("location") - if param_value is None and location: - # If no value is given, we try to get the file name directly from the URL - param_value = os.path.basename(location) if not value.type == "text": param_value = _split_if_str(param_value) if isinstance(value, galaxy.tools.parameters.basic.DataToolParameter): - if not isinstance(param_value, list): - param_value = [param_value] - for v in param_value: - _add_uploaded_dataset(context.for_state(), v, param_extra, value, required_files) + if location and value.multiple: + # We get the input/s from the location which can be a list of urls separated by commas + locations = _split_if_str(location) + param_value = [] + for location in locations: + v = os.path.basename(location) + param_value.append(v) + # param_extra should contain only the corresponding location + extra = dict(param_extra) + extra["location"] = location + _add_uploaded_dataset(context.for_state(), v, extra, value, required_files) + else: + if not isinstance(param_value, list): + param_value = [param_value] + for v in param_value: + _add_uploaded_dataset(context.for_state(), v, param_extra, value, required_files) processed_value = param_value elif isinstance(value, galaxy.tools.parameters.basic.DataCollectionToolParameter): assert "collection" in param_extra diff --git a/lib/galaxy/web_stack/__init__.py b/lib/galaxy/web_stack/__init__.py index ccaf263e5d0c..b30d7a924b5d 100644 --- a/lib/galaxy/web_stack/__init__.py +++ b/lib/galaxy/web_stack/__init__.py @@ -43,8 +43,9 @@ def get_app_kwds(cls, config_section, app_name=None, for_paste_app=False): return {} @classmethod - def register_postfork_function(cls, f, *args, **kwargs): - f(*args, **kwargs) + def register_postfork_function(cls, f, *args, post_fork_only=False, **kwargs): + if not post_fork_only: + f(*args, **kwargs) def __init__(self, app=None, config=None): self.app = app @@ -191,7 +192,7 @@ class GunicornApplicationStack(ApplicationStack): late_postfork_thread: threading.Thread @classmethod - def register_postfork_function(cls, f, *args, **kwargs): + def register_postfork_function(cls, f, *args, post_fork_only=False, **kwargs): # do_post_fork determines if we need to run postfork functions if cls.do_post_fork: # if so, we call ApplicationStack.late_postfork once after forking ... @@ -199,7 +200,7 @@ def register_postfork_function(cls, f, *args, **kwargs): os.register_at_fork(after_in_child=cls.late_postfork) # ... and store everything we need to run in ApplicationStack.postfork_functions cls.postfork_functions.append(lambda: f(*args, **kwargs)) - else: + elif not post_fork_only: f(*args, **kwargs) @classmethod @@ -300,8 +301,8 @@ def application_stack_log_formatter(): return logging.Formatter(fmt=application_stack_class().log_format) -def register_postfork_function(f, *args, **kwargs): - application_stack_class().register_postfork_function(f, *args, **kwargs) +def register_postfork_function(f, *args, post_fork_only=False, **kwargs): + application_stack_class().register_postfork_function(f, *args, post_fork_only=post_fork_only**kwargs) def get_app_kwds(config_section, app_name=None): diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py index 93ca937e0484..8a4e4af3dd73 100644 --- a/lib/galaxy/webapps/base/webapp.py +++ b/lib/galaxy/webapps/base/webapp.py @@ -306,7 +306,6 @@ def __init__( self.user_manager = app[UserManager] self.session_manager = app[GalaxySessionManager] super().__init__(environ) - self.expunge_all() config = self.app.config self.debug = asbool(config.get("debug", False)) x_frame_options = getattr(config, "x_frame_options", None) @@ -784,7 +783,7 @@ def _associate_user_history(self, user, prev_galaxy_session=None): history = None set_permissions = False try: - users_last_session = user.galaxy_sessions[0] + users_last_session = user.current_galaxy_session except Exception: users_last_session = None if ( diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index bd3c12900fe0..03ab15a0638f 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -838,7 +838,7 @@ def delete_typed( self, response: Response, trans: ProvidesHistoryContext = DependsOnTrans, - history_id: DecodedDatabaseIdField = HistoryIDPathParam, + history_id: str = Path(..., description="History ID or any string."), id: DecodedDatabaseIdField = HistoryItemIDPathParam, type: HistoryContentType = ContentTypePathParam, serialization_params: SerializationParams = Depends(query_serialization_params), diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 236da06dd71d..ec3e0f31d580 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -1219,12 +1219,6 @@ def wrap_in_middleware(app, global_conf, application_stack, **local_conf): normalize_remote_user_email=conf.get("normalize_remote_user_email", False), ), ) - # The recursive middleware allows for including requests in other - # requests or forwarding of requests, all on the server side. - if asbool(conf.get("use_recursive", True)): - from paste import recursive - - app = wrap_if_allowed(app, stack, recursive.RecursiveMiddleware, args=(conf,)) # Error middleware app = wrap_if_allowed(app, stack, ErrorMiddleware, args=(conf,)) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index 8cfacf0bf300..77142ca9669b 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -77,7 +77,7 @@ def get_value(self, trans, grid, user): class LastLoginColumn(grids.GridColumn): def get_value(self, trans, grid, user): if user.galaxy_sessions: - return self.format(user.galaxy_sessions[0].update_time) + return self.format(user.current_galaxy_session.update_time) return "never" def sort(self, trans, query, ascending, column_name=None): diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index 9d78f3767a97..fe75601e8858 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -253,13 +253,17 @@ def get_edit(self, trans, dataset_id=None, **kwd): # datatype changing datatype_options = [(ext_name, ext_id) for ext_id, ext_name in ldatatypes] datatype_disable = len(datatype_options) == 0 + datatype_input_default_value = None + current_datatype = trans.app.datatypes_registry.datatypes_by_extension.get(data.ext) + if current_datatype and current_datatype.is_datatype_change_allowed(): + datatype_input_default_value = data.ext datatype_inputs = [ { "type": "select", "name": "datatype", "label": "New Type", "options": datatype_options, - "value": [ext_id for ext_id, ext_name in ldatatypes if ext_id == data.ext], + "value": datatype_input_default_value, "help": "This will change the datatype of the existing dataset but not modify its contents. Use this if Galaxy has incorrectly guessed the type of your dataset.", } ] diff --git a/lib/galaxy/webapps/reports/buildapp.py b/lib/galaxy/webapps/reports/buildapp.py index cdc57f477aec..da3e74d3a93d 100644 --- a/lib/galaxy/webapps/reports/buildapp.py +++ b/lib/galaxy/webapps/reports/buildapp.py @@ -86,12 +86,6 @@ def wrap_in_middleware(app, global_conf, application_stack, **local_conf): # wrapped around the application (it can interact poorly with # other middleware): app = wrap_if_allowed(app, stack, httpexceptions.make_middleware, name="paste.httpexceptions", args=(conf,)) - # The recursive middleware allows for including requests in other - # requests or forwarding of requests, all on the server side. - if asbool(conf.get("use_recursive", True)): - from paste import recursive - - app = wrap_if_allowed(app, stack, recursive.RecursiveMiddleware, args=(conf,)) # Error middleware app = wrap_if_allowed(app, stack, ErrorMiddleware, args=(conf,)) diff --git a/lib/galaxy/webapps/reports/controllers/users.py b/lib/galaxy/webapps/reports/controllers/users.py index 543edb7979ac..bfc9cb233f83 100644 --- a/lib/galaxy/webapps/reports/controllers/users.py +++ b/lib/galaxy/webapps/reports/controllers/users.py @@ -170,8 +170,9 @@ def name_to_num(name): .filter(galaxy.model.User.table.c.deleted == false()) .order_by(galaxy.model.User.table.c.email) ): - if user.galaxy_sessions: - last_galaxy_session = user.galaxy_sessions[0] + current_galaxy_session = user.current_galaxy_session + if current_galaxy_session: + last_galaxy_session = current_galaxy_session if last_galaxy_session.update_time < cutoff_time: users.append((user.email, last_galaxy_session.update_time.strftime("%Y-%m-%d"))) else: diff --git a/lib/tool_shed/grids/admin_grids.py b/lib/tool_shed/grids/admin_grids.py index f38378677b37..45a4dcd1ff1f 100644 --- a/lib/tool_shed/grids/admin_grids.py +++ b/lib/tool_shed/grids/admin_grids.py @@ -46,8 +46,8 @@ def get_value(self, trans, grid, user): class LastLoginColumn(grids.GridColumn): def get_value(self, trans, grid, user): - if user.galaxy_sessions: - return self.format(user.galaxy_sessions[0].update_time) + if user.current_galaxy_session: + return self.format(user.current_galaxy_session.update_time) return "never" class StatusColumn(grids.GridColumn): diff --git a/lib/tool_shed/webapp/buildapp.py b/lib/tool_shed/webapp/buildapp.py index ef0e5d9cefe6..7565a7a2d98c 100644 --- a/lib/tool_shed/webapp/buildapp.py +++ b/lib/tool_shed/webapp/buildapp.py @@ -259,12 +259,7 @@ def wrap_in_middleware(app, global_conf, application_stack, **local_conf): normalize_remote_user_email=conf.get("normalize_remote_user_email", False), ), ) - # The recursive middleware allows for including requests in other - # requests or forwarding of requests, all on the server side. - if asbool(conf.get("use_recursive", True)): - from paste import recursive - app = wrap_if_allowed(app, stack, recursive.RecursiveMiddleware, args=(conf,)) # Transaction logging (apache access.log style) if asbool(conf.get("use_translogger", True)): from paste.translogger import TransLogger diff --git a/tools/interactive/interactivetool_isee.xml b/tools/interactive/interactivetool_isee.xml index 6074ae8a36d3..835f20479515 100644 --- a/tools/interactive/interactivetool_isee.xml +++ b/tools/interactive/interactivetool_isee.xml @@ -31,16 +31,6 @@ cp '$isee_script' '$outfile' &&