From 1d87385da43ab14007e9dd89be53363d8e0cd463 Mon Sep 17 00:00:00 2001 From: Laila Los <44241786+ElectronicBlueberry@users.noreply.github.com> Date: Wed, 13 Mar 2024 15:19:23 +0100 Subject: [PATCH 01/24] add index to parent_comment_id --- lib/galaxy/model/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 9495b3a82401..337e5cf7791a 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -7727,7 +7727,7 @@ class WorkflowStep(Base, RepresentById): uuid = Column(UUIDType) label = Column(Unicode(255)) temp_input_connections: Optional[InputConnDictType] - parent_comment_id = Column(Integer, ForeignKey("workflow_comment.id"), nullable=True) + parent_comment_id = Column(Integer, ForeignKey("workflow_comment.id"), index=True, nullable=True) parent_comment = relationship( "WorkflowComment", @@ -8174,7 +8174,7 @@ class WorkflowComment(Base, RepresentById): type = Column(String(16)) color = Column(String(16)) data = Column(JSONType) - parent_comment_id = Column(Integer, ForeignKey("workflow_comment.id"), nullable=True) + parent_comment_id = Column(Integer, ForeignKey("workflow_comment.id"), index=True, nullable=True) workflow = relationship( "Workflow", From bfef9d4f81f6bbfa494840932044a80bec4231e8 Mon Sep 17 00:00:00 2001 From: Laila Los <44241786+ElectronicBlueberry@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:41:05 +0100 Subject: [PATCH 02/24] 2dc3386d091f --- ...d091f_add_indexes_for_workflow_comment_.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 lib/galaxy/model/migrations/alembic/versions_gxy/2dc3386d091f_add_indexes_for_workflow_comment_.py diff --git a/lib/galaxy/model/migrations/alembic/versions_gxy/2dc3386d091f_add_indexes_for_workflow_comment_.py b/lib/galaxy/model/migrations/alembic/versions_gxy/2dc3386d091f_add_indexes_for_workflow_comment_.py new file mode 100644 index 000000000000..14c990e3086a --- /dev/null +++ b/lib/galaxy/model/migrations/alembic/versions_gxy/2dc3386d091f_add_indexes_for_workflow_comment_.py @@ -0,0 +1,44 @@ +"""add_indexes_for_workflow_comment_foreign_keys + +Revision ID: 2dc3386d091f +Revises: 8a19186a6ee7 +Create Date: 2024-03-13 15:25:52.587488 + +""" + +from galaxy.model.database_object_names import build_index_name +from galaxy.model.migrations.util import ( + create_index, + drop_index, +) + +# revision identifiers, used by Alembic. +revision = "2dc3386d091f" +down_revision = "8a19186a6ee7" +branch_labels = None +depends_on = None + +workflow_comment_table_name = "workflow_comment" +workflow_step_table_name = "workflow_step" +workflow_id_column_name = "workflow_id" +parent_comment_id_column_name = "parent_comment_id" + +workflow_step_parent_comment_index_name = build_index_name(workflow_step_table_name, parent_comment_id_column_name) +workflow_comment_workflow_id_index_name = build_index_name(workflow_comment_table_name, workflow_id_column_name) +workflow_comment_parent_comment_index_name = build_index_name( + workflow_comment_table_name, parent_comment_id_column_name +) + + +def upgrade(): + create_index(workflow_step_parent_comment_index_name, workflow_step_table_name, [parent_comment_id_column_name]) + create_index(workflow_comment_workflow_id_index_name, workflow_comment_table_name, [workflow_id_column_name]) + create_index( + workflow_comment_parent_comment_index_name, workflow_comment_table_name, [parent_comment_id_column_name] + ) + + +def downgrade(): + drop_index(workflow_step_parent_comment_index_name, workflow_step_table_name) + drop_index(workflow_comment_workflow_id_index_name, workflow_comment_table_name) + drop_index(workflow_comment_parent_comment_index_name, workflow_comment_table_name) From 3f1dc5ea8e75e585cd4218d1ef52d68e66bbb0cd Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 14 Mar 2024 12:57:05 +0100 Subject: [PATCH 03/24] Use correct history count instead of hid_counter --- client/src/components/Grid/configs/histories.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/Grid/configs/histories.ts b/client/src/components/Grid/configs/histories.ts index 678c55eaa993..8216cda71560 100644 --- a/client/src/components/Grid/configs/histories.ts +++ b/client/src/components/Grid/configs/histories.ts @@ -241,7 +241,7 @@ const fields: FieldArray = [ ], }, { - key: "hid_counter", + key: "count", title: "Items", type: "text", }, From f1e0779e9b14715a9578ff49663fc0f3be6f68a4 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:33:41 +0100 Subject: [PATCH 04/24] Improve published history listing in UI Initially load only summary views, as the additional details are requested for each history after loading anyway. --- client/src/components/Grid/configs/historiesPublished.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/client/src/components/Grid/configs/historiesPublished.ts b/client/src/components/Grid/configs/historiesPublished.ts index 23dc4bcccaff..9a1fd770391f 100644 --- a/client/src/components/Grid/configs/historiesPublished.ts +++ b/client/src/components/Grid/configs/historiesPublished.ts @@ -20,6 +20,7 @@ type SortKeyLiteral = "name" | "update_time" | undefined; */ async function getData(offset: number, limit: number, search: string, sort_by: string, sort_desc: boolean) { const { data, headers } = await historiesFetcher({ + view: "summary", limit, offset, search, From 5133abd79af570fe654c09539ca8c5a2064178b2 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 14 Mar 2024 13:24:43 +0100 Subject: [PATCH 05/24] Fix step type serialization for StoredWorkflowDetailed models Fixes https://github.com/galaxyproject/galaxy/issues/17713. --- client/src/api/schema/schema.ts | 99 ++++++++------------------------ lib/galaxy/managers/workflows.py | 6 +- lib/galaxy/schema/schema.py | 62 +++++++------------- lib/galaxy/schema/workflows.py | 7 ++- 4 files changed, 53 insertions(+), 121 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 44f5457df6bf..118ff0830cc1 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -6835,27 +6835,13 @@ export interface components { input_steps: { [key: string]: components["schemas"]["InputStep"] | undefined; }; - /** - * Tool ID - * @description The unique name of the tool associated with this step. - */ - tool_id?: string | null; - /** - * Tool Inputs - * @description TODO - */ - tool_inputs?: Record; - /** - * Tool Version - * @description The version of the tool associated with this step. - */ - tool_version?: string | null; /** * Type - * @description The type of workflow module. - * @default data_collection_input + * @constant */ - type?: components["schemas"]["WorkflowModuleType"]; + type: "data_collection_input"; + /** When */ + when: string | null; }; /** InputDataStep */ InputDataStep: { @@ -6876,27 +6862,13 @@ export interface components { input_steps: { [key: string]: components["schemas"]["InputStep"] | undefined; }; - /** - * Tool ID - * @description The unique name of the tool associated with this step. - */ - tool_id?: string | null; - /** - * Tool Inputs - * @description TODO - */ - tool_inputs?: Record; - /** - * Tool Version - * @description The version of the tool associated with this step. - */ - tool_version?: string | null; /** * Type - * @description The type of workflow module. - * @default data_input + * @constant */ - type?: components["schemas"]["WorkflowModuleType"]; + type: "data_input"; + /** When */ + when: string | null; }; /** InputParameterStep */ InputParameterStep: { @@ -6917,27 +6889,13 @@ export interface components { input_steps: { [key: string]: components["schemas"]["InputStep"] | undefined; }; - /** - * Tool ID - * @description The unique name of the tool associated with this step. - */ - tool_id?: string | null; - /** - * Tool Inputs - * @description TODO - */ - tool_inputs?: Record; - /** - * Tool Version - * @description The version of the tool associated with this step. - */ - tool_version?: string | null; /** * Type - * @description The type of workflow module. - * @default parameter_input + * @constant */ - type?: components["schemas"]["WorkflowModuleType"]; + type: "parameter"; + /** When */ + when: string | null; }; /** InputReferenceByLabel */ InputReferenceByLabel: { @@ -9705,10 +9663,11 @@ export interface components { }; /** * Type - * @description The type of workflow module. - * @default pause + * @constant */ - type?: components["schemas"]["WorkflowModuleType"]; + type: "pause"; + /** When */ + when: string | null; }; /** Person */ Person: { @@ -11111,10 +11070,11 @@ export interface components { }; /** * Type - * @description The type of workflow module. - * @default subworkflow + * @constant */ - type?: components["schemas"]["WorkflowModuleType"]; + type: "subworkflow"; + /** When */ + when: string | null; /** * Workflow ID * @description The encoded ID of the workflow that will be run on this step. @@ -11301,10 +11261,11 @@ export interface components { tool_version?: string | null; /** * Type - * @description The type of workflow module. - * @default tool + * @constant */ - type?: components["schemas"]["WorkflowModuleType"]; + type: "tool"; + /** When */ + when: string | null; }; /** Tour */ Tour: { @@ -12303,18 +12264,6 @@ export interface components { [key: string]: number | undefined; }; }; - /** - * WorkflowModuleType - * @description Available types of modules that represent a step in a Workflow. - * @enum {string} - */ - WorkflowModuleType: - | "data_input" - | "data_collection_input" - | "parameter_input" - | "subworkflow" - | "tool" - | "pause"; /** WriteInvocationStoreToPayload */ WriteInvocationStoreToPayload: { /** diff --git a/lib/galaxy/managers/workflows.py b/lib/galaxy/managers/workflows.py index 43b65e6b0615..18a61ffba6c5 100644 --- a/lib/galaxy/managers/workflows.py +++ b/lib/galaxy/managers/workflows.py @@ -1615,9 +1615,9 @@ def callback(input, prefixed_name, **kwargs): def _workflow_to_dict_instance(self, trans, stored, workflow, legacy=True): encode = self.app.security.encode_id sa_session = self.app.model.context - item = stored.to_dict(view="element", value_mapper={"id": encode}) + item = stored.to_dict(view="element") item["name"] = workflow.name - item["url"] = trans.url_builder("workflow", id=item["id"]) + item["url"] = trans.url_builder("workflow", id=encode(stored.id)) item["owner"] = stored.user.username item["email_hash"] = md5_hash_str(stored.user.email) item["slug"] = stored.slug @@ -1668,7 +1668,7 @@ def _workflow_to_dict_instance(self, trans, stored, workflow, legacy=True): del step_dict["tool_id"] del step_dict["tool_version"] del step_dict["tool_inputs"] - step_dict["workflow_id"] = encode(step.subworkflow.id) + step_dict["workflow_id"] = step.subworkflow.id for conn in step.input_connections: step_id = step.id if legacy else step.order_index diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index ec96b40fede0..61ad7ec7824d 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -2099,7 +2099,7 @@ class JobFullDetails(JobDetails): class StoredWorkflowSummary(Model, WithModelClass): - id: DecodedDatabaseIdField + id: EncodedDatabaseIdField model_class: STORED_WORKFLOW_MODEL_CLASS = ModelClassField(STORED_WORKFLOW_MODEL_CLASS) create_time: datetime = CreateTimeField update_time: datetime = UpdateTimeField @@ -2203,73 +2203,51 @@ class InputStep(Model): ) -class WorkflowModuleType(str, Enum): - """Available types of modules that represent a step in a Workflow.""" - - data_input = "data_input" - data_collection_input = "data_collection_input" - parameter_input = "parameter_input" - subworkflow = "subworkflow" - tool = "tool" - pause = "pause" # Experimental - - class WorkflowStepBase(Model): id: int = Field( ..., title="ID", description="The identifier of the step. It matches the index order of the step inside the workflow.", ) - type: WorkflowModuleType = Field(..., title="Type", description="The type of workflow module.") annotation: Optional[str] = AnnotationField input_steps: Dict[str, InputStep] = Field( ..., title="Input Steps", description="A dictionary containing information about the inputs connected to this workflow step.", ) + when: Optional[str] -class ToolBasedWorkflowStep(WorkflowStepBase): - tool_id: Optional[str] = Field( - None, title="Tool ID", description="The unique name of the tool associated with this step." - ) - tool_version: Optional[str] = Field( - None, title="Tool Version", description="The version of the tool associated with this step." - ) - tool_inputs: Any = Field(None, title="Tool Inputs", description="TODO") +class InputDataStep(WorkflowStepBase): + type: Literal["data_input"] -class InputDataStep(ToolBasedWorkflowStep): - type: WorkflowModuleType = Field( - WorkflowModuleType.data_input, title="Type", description="The type of workflow module." - ) +class InputDataCollectionStep(WorkflowStepBase): + type: Literal["data_collection_input"] -class InputDataCollectionStep(ToolBasedWorkflowStep): - type: WorkflowModuleType = Field( - WorkflowModuleType.data_collection_input, title="Type", description="The type of workflow module." - ) - - -class InputParameterStep(ToolBasedWorkflowStep): - type: WorkflowModuleType = Field( - WorkflowModuleType.parameter_input, title="Type", description="The type of workflow module." - ) +class InputParameterStep(WorkflowStepBase): + type: Literal["parameter"] class PauseStep(WorkflowStepBase): - type: WorkflowModuleType = Field(WorkflowModuleType.pause, title="Type", description="The type of workflow module.") + type: Literal["pause"] -class ToolStep(ToolBasedWorkflowStep): - type: WorkflowModuleType = Field(WorkflowModuleType.tool, title="Type", description="The type of workflow module.") +class ToolStep(WorkflowStepBase): + type: Literal["tool"] + tool_id: Optional[str] = Field( + None, title="Tool ID", description="The unique name of the tool associated with this step." + ) + tool_version: Optional[str] = Field( + None, title="Tool Version", description="The version of the tool associated with this step." + ) + tool_inputs: Any = Field(None, title="Tool Inputs", description="TODO") class SubworkflowStep(WorkflowStepBase): - type: WorkflowModuleType = Field( - WorkflowModuleType.subworkflow, title="Type", description="The type of workflow module." - ) - workflow_id: DecodedDatabaseIdField = Field( + type: Literal["subworkflow"] + workflow_id: EncodedDatabaseIdField = Field( ..., title="Workflow ID", description="The encoded ID of the workflow that will be run on this step." ) diff --git a/lib/galaxy/schema/workflows.py b/lib/galaxy/schema/workflows.py index 598878d749c2..cbc5379e49e6 100644 --- a/lib/galaxy/schema/workflows.py +++ b/lib/galaxy/schema/workflows.py @@ -197,7 +197,12 @@ class StoredWorkflowDetailed(StoredWorkflowSummary): ToolStep, SubworkflowStep, ], - ] = Field({}, title="Steps", description="A dictionary with information about all the steps of the workflow.") + ] = Field( + {}, + title="Steps", + description="A dictionary with information about all the steps of the workflow.", + discriminator="type", + ) importable: Optional[bool] = Field( ..., title="Importable", From d5a829d6d52b07eb1d5e34fffacbb7f46a7b0a80 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:50:53 +0100 Subject: [PATCH 06/24] Fix missing default value warning --- client/src/components/Grid/GridHistory.vue | 1 + 1 file changed, 1 insertion(+) diff --git a/client/src/components/Grid/GridHistory.vue b/client/src/components/Grid/GridHistory.vue index a54e2437d3be..1f23b2e2084b 100644 --- a/client/src/components/Grid/GridHistory.vue +++ b/client/src/components/Grid/GridHistory.vue @@ -24,6 +24,7 @@ interface Props { const props = withDefaults(defineProps(), { activeList: "my", + username: undefined, }); From babdcd9b031743a739c70ac92452ff70f4b865df Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:51:46 +0100 Subject: [PATCH 07/24] Display login required in tab when anonymous access archived histories --- client/src/components/Grid/GridHistory.vue | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/client/src/components/Grid/GridHistory.vue b/client/src/components/Grid/GridHistory.vue index 1f23b2e2084b..76229f380ea8 100644 --- a/client/src/components/Grid/GridHistory.vue +++ b/client/src/components/Grid/GridHistory.vue @@ -66,8 +66,16 @@ const props = withDefaults(defineProps(), { Public Histories - + Archived Histories + From c8777e45d453671206b3d22a4ae9f037e7ba4387 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:52:17 +0100 Subject: [PATCH 08/24] Redirect anonymous users when directly accessing histories/archived --- client/src/entry/analysis/router.js | 1 + 1 file changed, 1 insertion(+) diff --git a/client/src/entry/analysis/router.js b/client/src/entry/analysis/router.js index aa9e540be7d6..2e93e6823d37 100644 --- a/client/src/entry/analysis/router.js +++ b/client/src/entry/analysis/router.js @@ -291,6 +291,7 @@ export function getRouter(Galaxy) { props: { activeList: "archived", }, + redirect: redirectAnon(), }, { path: "histories/list", From b3bce86f305a64108d1ffe5e1ea7ff51a6f04d95 Mon Sep 17 00:00:00 2001 From: Laila Los <44241786+ElectronicBlueberry@users.noreply.github.com> Date: Thu, 14 Mar 2024 14:24:47 +0100 Subject: [PATCH 09/24] remove side-effect and add lastQueue --- .../CurrentCollection/CollectionPanel.vue | 11 +++- client/src/stores/collectionElementsStore.ts | 52 +++++++++++++------ 2 files changed, 45 insertions(+), 18 deletions(-) diff --git a/client/src/components/History/CurrentCollection/CollectionPanel.vue b/client/src/components/History/CurrentCollection/CollectionPanel.vue index 2f58d1356e2a..cd83d5fbc56b 100644 --- a/client/src/components/History/CurrentCollection/CollectionPanel.vue +++ b/client/src/components/History/CurrentCollection/CollectionPanel.vue @@ -43,7 +43,16 @@ const dsc = computed(() => { } return currentCollection; }); -const collectionElements = computed(() => collectionElementsStore.getCollectionElements(dsc.value, offset.value)); + +watch( + () => [dsc.value, offset.value], + () => { + collectionElementsStore.fetchMissingElements(dsc.value, offset.value); + }, + { immediate: true } +); + +const collectionElements = computed(() => collectionElementsStore.getCollectionElements(dsc.value)); const loading = computed(() => collectionElementsStore.isLoadingCollectionElements(dsc.value)); const jobState = computed(() => ("job_state_summary" in dsc.value ? dsc.value.job_state_summary : undefined)); const populatedStateMsg = computed(() => diff --git a/client/src/stores/collectionElementsStore.ts b/client/src/stores/collectionElementsStore.ts index ddeb60102012..eb5dee564504 100644 --- a/client/src/stores/collectionElementsStore.ts +++ b/client/src/stores/collectionElementsStore.ts @@ -4,6 +4,8 @@ import { computed, del, ref, set } from "vue"; import type { CollectionEntry, DCESummary, HDCASummary, HistoryContentItemBase } from "@/api"; import { isHDCA } from "@/api"; import { fetchCollectionDetails, fetchElementsFromCollection } from "@/api/datasetCollections"; +import { mergeArray } from "@/store/historyStore/model/utilities"; +import { ActionSkippedError, LastQueue } from "@/utils/lastQueue"; /** * Represents an element in a collection that has not been fetched yet. @@ -46,10 +48,9 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore", } const getCollectionElements = computed(() => { - return (collection: CollectionEntry, offset = 0, limit = FETCH_LIMIT) => { + return (collection: CollectionEntry) => { const storedElements = storedCollectionElements.value[getCollectionKey(collection)] ?? initWithPlaceholderElements(collection); - fetchMissingElements({ collection, storedElements, offset, limit }); return storedElements; }; }); @@ -60,17 +61,22 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore", }; }); - async function fetchMissingElements(params: { + const lastQueue = new LastQueue(1000, true); + + type FetchParams = { collection: CollectionEntry; - storedElements: DCEEntry[]; offset: number; limit: number; - }) { - const collectionKey = getCollectionKey(params.collection); + }; + + async function fetchMissing({ collection, offset, limit = FETCH_LIMIT }: FetchParams) { + const collectionKey = getCollectionKey(collection); + const storedElements = getCollectionElements.value(collection); + try { // We should fetch only missing (placeholder) elements from the range - const firstMissingIndexInRange = params.storedElements - .slice(params.offset, params.offset + params.limit) + const firstMissingIndexInRange = storedElements + .slice(offset, offset + limit) .findIndex((element) => isPlaceholder(element) && !element.fetching); if (firstMissingIndexInRange === -1) { @@ -78,26 +84,37 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore", return; } // Adjust the offset to the first missing element - params.offset += firstMissingIndexInRange; + offset += firstMissingIndexInRange; set(loadingCollectionElements.value, collectionKey, true); // Mark all elements in the range as fetching - params.storedElements - .slice(params.offset, params.offset + params.limit) + storedElements + .slice(offset, offset + limit) .forEach((element) => isPlaceholder(element) && (element.fetching = true)); + const fetchedElements = await fetchElementsFromCollection({ - entry: params.collection, - offset: params.offset, - limit: params.limit, + entry: collection, + offset: offset, + limit: limit, }); - // Update only the elements that were fetched - params.storedElements.splice(params.offset, fetchedElements.length, ...fetchedElements); - set(storedCollectionElements.value, collectionKey, params.storedElements); + + return fetchedElements; } finally { del(loadingCollectionElements.value, collectionKey); } } + async function fetchMissingElements(collection: CollectionEntry, offset: number) { + try { + const elements = await lastQueue.enqueue(fetchMissing, { collection, offset }); + mergeArray(getCollectionKey(collection), elements, storedCollectionElements.value, "id"); + } catch (e) { + if (!(e instanceof ActionSkippedError)) { + throw e; + } + } + } + async function loadCollectionElements(collection: CollectionEntry) { const elements = await fetchElementsFromCollection({ entry: collection }); set(storedCollectionElements.value, getCollectionKey(collection), elements); @@ -154,5 +171,6 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore", loadCollectionElements, saveCollections, getCollectionKey, + fetchMissingElements, }; }); From 1799e7c4d606cf3e74ce7986b713b704645f7b3e Mon Sep 17 00:00:00 2001 From: guerler Date: Thu, 14 Mar 2024 16:35:28 +0300 Subject: [PATCH 10/24] Always display grid pagination on the right hand side, regardless of batch operation availability --- client/src/components/Grid/GridList.vue | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/client/src/components/Grid/GridList.vue b/client/src/components/Grid/GridList.vue index da08de2739c5..16016ec5fa7f 100644 --- a/client/src/components/Grid/GridList.vue +++ b/client/src/components/Grid/GridList.vue @@ -363,7 +363,7 @@ watch(operationMessage, () => {