From cf99e2556bf05ab077654fca92f32b199cb38643 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 6 Dec 2023 11:57:13 +0100 Subject: [PATCH 1/6] Add missing response model to API endpoint --- client/src/api/schema/schema.ts | 23 ++++++++++++++++++- lib/galaxy/webapps/galaxy/api/datasets.py | 3 ++- .../webapps/galaxy/services/datasets.py | 6 +++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 9988744c92f9..ea2cd1f83950 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -3502,6 +3502,11 @@ export interface components { */ error_message: string; }; + /** + * DatasetExtraFiles + * @description A list of extra files associated with a dataset. + */ + DatasetExtraFiles: components["schemas"]["ExtraFileEntry"][]; /** * DatasetInheritanceChain * @default [] @@ -4370,6 +4375,16 @@ export interface components { }; /** ExportTaskListResponse */ ExportTaskListResponse: components["schemas"]["ObjectExportTaskResponse"][]; + /** ExtraFileEntry */ + ExtraFileEntry: { + /** @description The class of this entry, either File or Directory. */ + class: components["schemas"]["ExtraFilesEntryClass"]; + /** + * Path + * @description Relative path to the file or directory. + */ + path: string; + }; /** ExtraFiles */ ExtraFiles: { /** @@ -4382,6 +4397,12 @@ export interface components { items_from?: string; src: components["schemas"]["Src"]; }; + /** + * ExtraFilesEntryClass + * @description An enumeration. + * @enum {string} + */ + ExtraFilesEntryClass: "Directory" | "File"; /** FavoriteObject */ FavoriteObject: { /** @@ -13761,7 +13782,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": Record; + "application/json": components["schemas"]["DatasetExtraFiles"]; }; }; /** @description Validation Error */ diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index 46da439b7b41..fd77ff4d576f 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -57,6 +57,7 @@ ComputeDatasetHashPayload, ConvertedDatasetsMap, DatasetContentType, + DatasetExtraFiles, DatasetInheritanceChain, DatasetsService, DatasetStorageDetails, @@ -249,7 +250,7 @@ def extra_files( trans=DependsOnTrans, history_id: DecodedDatabaseIdField = HistoryIDPathParam, history_content_id: DecodedDatabaseIdField = DatasetIDPathParam, - ): + ) -> DatasetExtraFiles: return self.service.extra_files(trans, history_content_id) @router.get( diff --git a/lib/galaxy/webapps/galaxy/services/datasets.py b/lib/galaxy/webapps/galaxy/services/datasets.py index 4a40e2e2576a..2e20d689dea4 100644 --- a/lib/galaxy/webapps/galaxy/services/datasets.py +++ b/lib/galaxy/webapps/galaxy/services/datasets.py @@ -177,6 +177,12 @@ class ExtraFileEntry(Model): ) +class DatasetExtraFiles(Model): + """A list of extra files associated with a dataset.""" + + __root__: List[ExtraFileEntry] + + class DatasetTextContentDetails(Model): item_data: Optional[str] = Field( description="First chunk of text content (maximum 1MB) of the dataset.", From ed416c0b37d49a1a09efba0f2c216dca5e2b75fd Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 6 Dec 2023 12:29:55 +0100 Subject: [PATCH 2/6] Add alias endpoint for extra_files - Deprecate the endpoint requiring a `history_id` as it is not necessary --- client/src/api/schema/schema.ts | 47 +++++++++++++++++++++-- lib/galaxy/webapps/galaxy/api/datasets.py | 27 ++++++++----- 2 files changed, 61 insertions(+), 13 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index ea2cd1f83950..5153606ca0ef 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -155,6 +155,10 @@ export interface paths { */ get: operations["converted_ext_api_datasets__dataset_id__converted__ext__get"]; }; + "/api/datasets/{dataset_id}/extra_files": { + /** Get the list of extra files/directories associated with a dataset. */ + get: operations["extra_files_api_datasets__dataset_id__extra_files_get"]; + }; "/api/datasets/{dataset_id}/get_content_as_text": { /** Returns dataset content as Text. */ get: operations["get_content_as_text_api_datasets__dataset_id__get_content_as_text_get"]; @@ -650,8 +654,12 @@ export interface paths { head: operations["history_contents_display_api_histories__history_id__contents__history_content_id__display_head"]; }; "/api/histories/{history_id}/contents/{history_content_id}/extra_files": { - /** Generate list of extra files. */ - get: operations["extra_files_api_histories__history_id__contents__history_content_id__extra_files_get"]; + /** + * Get the list of extra files/directories associated with a dataset. + * @deprecated + * @description This endpoint is deprecated, please use `/api/datasets/{dataset_id}/extra_files` instead. + */ + get: operations["extra_files_history_api_histories__history_id__contents__history_content_id__extra_files_get"]; }; "/api/histories/{history_id}/contents/{history_content_id}/metadata_file": { /** Returns the metadata file associated with this history item. */ @@ -10840,6 +10848,33 @@ export interface operations { }; }; }; + extra_files_api_datasets__dataset_id__extra_files_get: { + /** Get the list of extra files/directories associated with a dataset. */ + parameters: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string; + }; + /** @description The encoded database identifier of the dataset. */ + path: { + dataset_id: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["DatasetExtraFiles"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_content_as_text_api_datasets__dataset_id__get_content_as_text_get: { /** Returns dataset content as Text. */ parameters: { @@ -13764,8 +13799,12 @@ export interface operations { }; }; }; - extra_files_api_histories__history_id__contents__history_content_id__extra_files_get: { - /** Generate list of extra files. */ + extra_files_history_api_histories__history_id__contents__history_content_id__extra_files_get: { + /** + * Get the list of extra files/directories associated with a dataset. + * @deprecated + * @description This endpoint is deprecated, please use `/api/datasets/{dataset_id}/extra_files` instead. + */ parameters: { /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index fd77ff4d576f..ebc21fbd27a4 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -73,7 +73,7 @@ DatasetIDPathParam: DecodedDatabaseIdField = Path(..., description="The encoded database identifier of the dataset.") -HistoryIDPathParam: DecodedDatabaseIdField = Path(..., description="The encoded database identifier of the History.") +HistoryIDPathParam = Path(description="The encoded database identifier of the History.") DatasetSourceQueryParam: DatasetSourceType = Query( default=DatasetSourceType.hda, @@ -242,17 +242,30 @@ def update_permissions( @router.get( "/api/histories/{history_id}/contents/{history_content_id}/extra_files", - summary="Generate list of extra files.", + summary="Get the list of extra files/directories associated with a dataset.", tags=["histories"], + deprecated=True, ) - def extra_files( + def extra_files_history( self, trans=DependsOnTrans, history_id: DecodedDatabaseIdField = HistoryIDPathParam, history_content_id: DecodedDatabaseIdField = DatasetIDPathParam, ) -> DatasetExtraFiles: + """This endpoint is deprecated, please use `/api/datasets/{dataset_id}/extra_files` instead.""" return self.service.extra_files(trans, history_content_id) + @router.get( + "/api/datasets/{dataset_id}/extra_files", + summary="Get the list of extra files/directories associated with a dataset.", + ) + def extra_files( + self, + trans=DependsOnTrans, + dataset_id: DecodedDatabaseIdField = DatasetIDPathParam, + ) -> DatasetExtraFiles: + return self.service.extra_files(trans, dataset_id) + @router.get( "/api/histories/{history_id}/contents/{history_content_id}/display", name="history_contents_display", @@ -270,9 +283,7 @@ def display_history_content( self, request: Request, trans=DependsOnTrans, - history_id: Optional[DecodedDatabaseIdField] = Path( - description="The encoded database identifier of the History.", - ), + history_id: Optional[DecodedDatabaseIdField] = HistoryIDPathParam, history_content_id: DecodedDatabaseIdField = DatasetIDPathParam, preview: bool = PreviewQueryParam, filename: Optional[str] = FilenameQueryParam, @@ -357,9 +368,7 @@ def _display( def get_metadata_file_history_content( self, trans=DependsOnTrans, - history_id: DecodedDatabaseIdField = Path( - description="The encoded database identifier of the History.", - ), + history_id: DecodedDatabaseIdField = HistoryIDPathParam, history_content_id: DecodedDatabaseIdField = DatasetIDPathParam, metadata_file: str = Query( ..., From ff64638a007d5ec70741f7bd6a44f67cf97076fe Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 7 Dec 2023 12:07:07 +0100 Subject: [PATCH 3/6] Add dataset extra files Pinia store --- client/src/api/datasets.ts | 5 ++- client/src/stores/datasetExtraFilesStore.ts | 46 +++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 client/src/stores/datasetExtraFilesStore.ts diff --git a/client/src/api/datasets.ts b/client/src/api/datasets.ts index 09948b45ce26..ded071c812af 100644 --- a/client/src/api/datasets.ts +++ b/client/src/api/datasets.ts @@ -1,7 +1,7 @@ import type { FetchArgType } from "openapi-typescript-fetch"; import { DatasetDetails } from "@/api"; -import { fetcher } from "@/api/schema"; +import { components, fetcher } from "@/api/schema"; import { withPrefix } from "@/utils/redirect"; export const datasetsFetcher = fetcher.path("/api/datasets").method("get").create(); @@ -87,3 +87,6 @@ export async function copyDataset( export function getCompositeDatasetLink(historyDatasetId: string, path: string) { return withPrefix(`/api/datasets/${historyDatasetId}/display?filename=${path}`); } + +export type DatasetExtraFiles = components["schemas"]["DatasetExtraFiles"]; +export const fetchDatasetExtraFiles = fetcher.path("/api/datasets/{dataset_id}/extra_files").method("get").create(); diff --git a/client/src/stores/datasetExtraFilesStore.ts b/client/src/stores/datasetExtraFilesStore.ts new file mode 100644 index 000000000000..14080fd978f5 --- /dev/null +++ b/client/src/stores/datasetExtraFilesStore.ts @@ -0,0 +1,46 @@ +import { defineStore } from "pinia"; +import { computed, del, ref, set } from "vue"; + +import { DatasetExtraFiles, fetchDatasetExtraFiles } from "@/api/datasets"; + +export const useDatasetExtraFilesStore = defineStore("datasetExtraFilesStore", () => { + const storedDatasetExtraFiles = ref<{ [key: string]: DatasetExtraFiles }>({}); + const loading = ref<{ [key: string]: boolean }>({}); + + const getDatasetExtraFiles = computed(() => { + return (datasetId: string) => { + const datasetExtFiles = storedDatasetExtraFiles.value[datasetId]; + if (!datasetExtFiles && !loading.value[datasetId]) { + fetchDatasetExtFilesByDatasetId({ id: datasetId }); + } + return datasetExtFiles ?? null; + }; + }); + + const isLoadingDatasetExtraFiles = computed(() => { + return (datasetId: string) => { + return loading.value[datasetId] ?? false; + }; + }); + + async function fetchDatasetExtFilesByDatasetId(params: { id: string }) { + const datasetId = params.id; + set(loading.value, datasetId, true); + try { + const { data: datasetExtFiles } = await fetchDatasetExtraFiles({ + dataset_id: datasetId, + }); + set(storedDatasetExtraFiles.value, datasetId, datasetExtFiles); + return datasetExtFiles; + } finally { + del(loading.value, datasetId); + } + } + + return { + storedDatasetExtraFiles, + getDatasetExtraFiles, + isLoadingDatasetExtraFiles, + fetchDatasetExtFilesByDatasetId, + }; +}); From 6eb95029db158c20dc52fe4664b3684a0fd5973a Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 7 Dec 2023 12:09:21 +0100 Subject: [PATCH 4/6] Refactor compositeDatasetUtils - Use new Pinia store - Convert to typescript - Simplify code --- .../Dataset/compositeDatasetUtils.js | 36 --------------- .../Dataset/compositeDatasetUtils.ts | 45 +++++++++++++++++++ 2 files changed, 45 insertions(+), 36 deletions(-) delete mode 100644 client/src/components/Dataset/compositeDatasetUtils.js create mode 100644 client/src/components/Dataset/compositeDatasetUtils.ts diff --git a/client/src/components/Dataset/compositeDatasetUtils.js b/client/src/components/Dataset/compositeDatasetUtils.js deleted file mode 100644 index 087f61ad9696..000000000000 --- a/client/src/components/Dataset/compositeDatasetUtils.js +++ /dev/null @@ -1,36 +0,0 @@ -import { getCompositeDatasetLink } from "@/api/datasets"; - -import store from "../../store/index"; - -export const getPathDestination = async (history_dataset_id, path) => { - const computePathDestination = (pathDestination) => { - if (path === undefined || path === "undefined") { - return pathDestination; - } - - const filepath = path; - - const datasetEntry = datasetContent.find((datasetEntry) => { - return filepath === datasetEntry.path; - }); - - if (datasetEntry) { - if (datasetEntry.class === "Directory") { - pathDestination.isDirectory = true; - pathDestination.filepath = filepath; - return pathDestination; - } - pathDestination.fileLink = getCompositeDatasetLink(history_dataset_id, datasetEntry.path); - } - return pathDestination; - }; - - let datasetContent = store.getters.getDatasetExtFiles(history_dataset_id); - - if (datasetContent == null) { - await store.dispatch("fetchDatasetExtFiles", history_dataset_id); - datasetContent = store.getters.getDatasetExtFiles(history_dataset_id); - } - - return computePathDestination({ datasetContent: datasetContent }); -}; diff --git a/client/src/components/Dataset/compositeDatasetUtils.ts b/client/src/components/Dataset/compositeDatasetUtils.ts new file mode 100644 index 000000000000..763bb846306a --- /dev/null +++ b/client/src/components/Dataset/compositeDatasetUtils.ts @@ -0,0 +1,45 @@ +import { DatasetExtraFiles, getCompositeDatasetLink } from "@/api/datasets"; +import { useDatasetExtraFilesStore } from "@/stores/datasetExtraFilesStore"; + +interface PathDestination { + datasetContent: DatasetExtraFiles; + isDirectory: boolean; + filepath?: string; + fileLink?: string; +} + +export async function getPathDestination(dataset_id: string, path?: string): Promise { + const datasetExtraFilesStore = useDatasetExtraFilesStore(); + + let datasetExtraFiles = datasetExtraFilesStore.getDatasetExtraFiles(dataset_id); + if (!datasetExtraFiles) { + await datasetExtraFilesStore.fetchDatasetExtFilesByDatasetId({ id: dataset_id }); + datasetExtraFiles = datasetExtraFilesStore.getDatasetExtraFiles(dataset_id); + } + + if (datasetExtraFiles === null) { + return null; + } + + const pathDestination: PathDestination = { datasetContent: datasetExtraFiles, isDirectory: false, filepath: path }; + + if (path === undefined || path === "undefined") { + return pathDestination; + } + + const filepath = path; + + const datasetEntry = datasetExtraFiles?.find((entry) => { + return filepath === entry.path; + }); + + if (datasetEntry) { + if (datasetEntry.class === "Directory") { + pathDestination.isDirectory = true; + pathDestination.filepath = filepath; + return pathDestination; + } + pathDestination.fileLink = getCompositeDatasetLink(dataset_id, datasetEntry.path); + } + return pathDestination; +} From e41447b6331788588221c055ecf58d0a08720532 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 7 Dec 2023 12:13:50 +0100 Subject: [PATCH 5/6] Remove old Vuex store --- client/src/store/datasetExtFilesStore.js | 35 ------------------------ client/src/store/index.js | 2 -- 2 files changed, 37 deletions(-) delete mode 100644 client/src/store/datasetExtFilesStore.js diff --git a/client/src/store/datasetExtFilesStore.js b/client/src/store/datasetExtFilesStore.js deleted file mode 100644 index 8ae400479093..000000000000 --- a/client/src/store/datasetExtFilesStore.js +++ /dev/null @@ -1,35 +0,0 @@ -import axios from "axios"; -import { getAppRoot } from "onload/loadConfig"; -import Vue from "vue"; - -export const state = { - datasetExtFilesById: {}, -}; - -const getters = { - getDatasetExtFiles: (state) => (history_dataset_id) => { - return state.datasetExtFilesById[history_dataset_id] || null; - }, -}; - -const actions = { - fetchDatasetExtFiles: async ({ commit }, history_dataset_id) => { - const { data } = await axios.get( - `${getAppRoot()}api/histories/${history_dataset_id}/contents/${history_dataset_id}/extra_files` - ); - commit("saveDatasetExtFiles", { history_dataset_id, datasetExtFiles: data }); - }, -}; - -const mutations = { - saveDatasetExtFiles: (state, { history_dataset_id, datasetExtFiles }) => { - Vue.set(state.datasetExtFilesById, history_dataset_id, datasetExtFiles); - }, -}; - -export const datasetExtFilesStore = { - state, - getters, - actions, - mutations, -}; diff --git a/client/src/store/index.js b/client/src/store/index.js index fb33a9ab671b..fd6f6d7bde87 100644 --- a/client/src/store/index.js +++ b/client/src/store/index.js @@ -9,7 +9,6 @@ import Vuex from "vuex"; import createCache from "vuex-cache"; import VuexPersistence from "vuex-persist"; -import { datasetExtFilesStore } from "./datasetExtFilesStore"; import { datasetPathDestinationStore } from "./datasetPathDestinationStore"; import { gridSearchStore } from "./gridSearchStore"; import { invocationStore } from "./invocationStore"; @@ -41,7 +40,6 @@ export function createStore() { const storeConfig = { plugins: [createCache(), panelsPersistence.plugin], modules: { - datasetExtFiles: datasetExtFilesStore, datasetPathDestination: datasetPathDestinationStore, invocations: invocationStore, gridSearch: gridSearchStore, From 31809d5e3f695f71078fc1b4ee08aecf158ba4ef Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 8 Dec 2023 22:17:01 +0100 Subject: [PATCH 6/6] Remove deprecation flag --- client/src/api/schema/schema.ts | 12 ++---------- lib/galaxy/webapps/galaxy/api/datasets.py | 2 -- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 5153606ca0ef..bd06205258ec 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -654,11 +654,7 @@ export interface paths { head: operations["history_contents_display_api_histories__history_id__contents__history_content_id__display_head"]; }; "/api/histories/{history_id}/contents/{history_content_id}/extra_files": { - /** - * Get the list of extra files/directories associated with a dataset. - * @deprecated - * @description This endpoint is deprecated, please use `/api/datasets/{dataset_id}/extra_files` instead. - */ + /** Get the list of extra files/directories associated with a dataset. */ get: operations["extra_files_history_api_histories__history_id__contents__history_content_id__extra_files_get"]; }; "/api/histories/{history_id}/contents/{history_content_id}/metadata_file": { @@ -13800,11 +13796,7 @@ export interface operations { }; }; extra_files_history_api_histories__history_id__contents__history_content_id__extra_files_get: { - /** - * Get the list of extra files/directories associated with a dataset. - * @deprecated - * @description This endpoint is deprecated, please use `/api/datasets/{dataset_id}/extra_files` instead. - */ + /** Get the list of extra files/directories associated with a dataset. */ parameters: { /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index ebc21fbd27a4..14d4bc1fe019 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -244,7 +244,6 @@ def update_permissions( "/api/histories/{history_id}/contents/{history_content_id}/extra_files", summary="Get the list of extra files/directories associated with a dataset.", tags=["histories"], - deprecated=True, ) def extra_files_history( self, @@ -252,7 +251,6 @@ def extra_files_history( history_id: DecodedDatabaseIdField = HistoryIDPathParam, history_content_id: DecodedDatabaseIdField = DatasetIDPathParam, ) -> DatasetExtraFiles: - """This endpoint is deprecated, please use `/api/datasets/{dataset_id}/extra_files` instead.""" return self.service.extra_files(trans, history_content_id) @router.get(