From 07b864b9d70e781e69e80b558bbe086b25b04feb Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 4 Jan 2021 22:46:44 -0500 Subject: [PATCH 1/2] Revise history import and export. - Provide a lot more context on what is and has happened to users during both import and export process. - Allow import and export to galaxy file plugins. --- .../src/components/FilesDialog/FilesInput.vue | 54 ++++++ .../src/components/History/HistoryDetails.vue | 2 +- .../components/HistoryExport/ExportLink.vue | 61 +++++++ .../components/HistoryExport/Index.test.js | 18 ++ client/src/components/HistoryExport/Index.vue | 39 ++++ .../components/HistoryExport/ToLink.test.js | 64 +++++++ .../src/components/HistoryExport/ToLink.vue | 166 ++++++++++++++++++ .../HistoryExport/ToRemoteFile.test.js | 72 ++++++++ .../components/HistoryExport/ToRemoteFile.vue | 119 +++++++++++++ client/src/components/HistoryExport/index.js | 1 + client/src/components/HistoryImport.test.js | 79 +++++++++ client/src/components/HistoryImport.vue | 157 ++++++++++++++--- .../components/JobInformation/JobError.vue | 70 ++++++++ .../JobInformation/JobInformation.vue | 18 ++ client/src/components/JobStates/wait.js | 29 +++ client/src/entry/analysis/AnalysisRouter.js | 8 + client/src/mvc/history/job-states-model.js | 8 +- client/src/mvc/history/options-menu.js | 7 +- client/src/utils/simple-error.js | 4 +- lib/galaxy/managers/histories.py | 47 +++++ lib/galaxy/model/__init__.py | 9 + lib/galaxy/webapps/base/controller.py | 3 +- lib/galaxy/webapps/galaxy/api/histories.py | 43 ++--- lib/galaxy/webapps/galaxy/buildapp.py | 4 + .../webapps/galaxy/controllers/history.py | 33 +--- 25 files changed, 1032 insertions(+), 83 deletions(-) create mode 100644 client/src/components/FilesDialog/FilesInput.vue create mode 100644 client/src/components/HistoryExport/ExportLink.vue create mode 100644 client/src/components/HistoryExport/Index.test.js create mode 100644 client/src/components/HistoryExport/Index.vue create mode 100644 client/src/components/HistoryExport/ToLink.test.js create mode 100644 client/src/components/HistoryExport/ToLink.vue create mode 100644 client/src/components/HistoryExport/ToRemoteFile.test.js create mode 100644 client/src/components/HistoryExport/ToRemoteFile.vue create mode 100644 client/src/components/HistoryExport/index.js create mode 100644 client/src/components/HistoryImport.test.js create mode 100644 client/src/components/JobInformation/JobError.vue create mode 100644 client/src/components/JobStates/wait.js diff --git a/client/src/components/FilesDialog/FilesInput.vue b/client/src/components/FilesDialog/FilesInput.vue new file mode 100644 index 000000000000..a56a184af73f --- /dev/null +++ b/client/src/components/FilesDialog/FilesInput.vue @@ -0,0 +1,54 @@ + + + diff --git a/client/src/components/History/HistoryDetails.vue b/client/src/components/History/HistoryDetails.vue index d85c0429fbd8..148cf9047e07 100644 --- a/client/src/components/History/HistoryDetails.vue +++ b/client/src/components/History/HistoryDetails.vue @@ -96,7 +96,7 @@ key="export-history-to-file" title="Export History to File" icon="fas fa-file-archive" - @click="iframeRedirect('/history/export_archive?preview=True')" + @click="backboneRoute(`/histories/${history.id}/export`)" /> diff --git a/client/src/components/HistoryExport/ExportLink.vue b/client/src/components/HistoryExport/ExportLink.vue new file mode 100644 index 000000000000..f42627f27455 --- /dev/null +++ b/client/src/components/HistoryExport/ExportLink.vue @@ -0,0 +1,61 @@ + + + diff --git a/client/src/components/HistoryExport/Index.test.js b/client/src/components/HistoryExport/Index.test.js new file mode 100644 index 000000000000..6d133bbbdcb2 --- /dev/null +++ b/client/src/components/HistoryExport/Index.test.js @@ -0,0 +1,18 @@ +import { shallowMount } from "@vue/test-utils"; +import Index from "./Index.vue"; +import { getLocalVue } from "jest/helpers"; + +const localVue = getLocalVue(); + +describe("Index.vue", () => { + it("should render tabs", () => { + // just make sure the component renders to catch obvious big errors + const wrapper = shallowMount(Index, { + propsData: { + historyId: "test_id", + }, + localVue, + }); + expect(wrapper.exists("b-tabs-stub")).toBeTruthy(); + }); +}); diff --git a/client/src/components/HistoryExport/Index.vue b/client/src/components/HistoryExport/Index.vue new file mode 100644 index 000000000000..3c9f2bb1e503 --- /dev/null +++ b/client/src/components/HistoryExport/Index.vue @@ -0,0 +1,39 @@ + + + diff --git a/client/src/components/HistoryExport/ToLink.test.js b/client/src/components/HistoryExport/ToLink.test.js new file mode 100644 index 000000000000..39b3defb70f1 --- /dev/null +++ b/client/src/components/HistoryExport/ToLink.test.js @@ -0,0 +1,64 @@ +import { shallowMount } from "@vue/test-utils"; +import { getLocalVue } from "jest/helpers"; +import ToLink from "./ToLink.vue"; +import flushPromises from "flush-promises"; +import MockAdapter from "axios-mock-adapter"; +import axios from "axios"; +import { waitOnJob } from "components/JobStates/wait"; + +const localVue = getLocalVue(); +const TEST_HISTORY_ID = "hist1235"; +const TEST_EXPORTS_URL = `/api/histories/${TEST_HISTORY_ID}/exports`; +const TEST_JOB_ID = "test1234job"; + +jest.mock("components/JobStates/wait"); + +describe("ToLink.vue", () => { + let axiosMock; + let wrapper; + + async function mountWithInitialExports(exports) { + axiosMock.onGet(TEST_EXPORTS_URL).reply(200, exports); + wrapper = shallowMount(ToLink, { + propsData: { + historyId: TEST_HISTORY_ID, + }, + localVue, + }); + await wrapper.vm.$nextTick(); + expect(wrapper.find("loading-span-stub").exists()).toBeTruthy(); + await flushPromises(); + } + + beforeEach(async () => { + axiosMock = new MockAdapter(axios); + }); + + it("should display a link if no exports ever generated", async () => { + await mountWithInitialExports([]); + expect(wrapper.find(".export-link")).toBeTruthy(); + expect(wrapper.find("loading-span-stub").exists()).toBeFalsy(); // loading span gone + }); + + it("should start polling if latest export is preparing", async () => { + let then = null; + waitOnJob.mockReturnValue( + new Promise((then_) => { + then = then_; + }) + ); + await mountWithInitialExports([ + { + preparing: true, + job_id: TEST_JOB_ID, + }, + ]); + expect(then).toBeTruthy(); + expect(wrapper.vm.waitingOnJob).toBeTruthy(); + expect(wrapper.find("loading-span-stub").exists()).toBeTruthy(); + }); + + afterEach(() => { + axiosMock.restore(); + }); +}); diff --git a/client/src/components/HistoryExport/ToLink.vue b/client/src/components/HistoryExport/ToLink.vue new file mode 100644 index 000000000000..8cb5f8fc036d --- /dev/null +++ b/client/src/components/HistoryExport/ToLink.vue @@ -0,0 +1,166 @@ + + + diff --git a/client/src/components/HistoryExport/ToRemoteFile.test.js b/client/src/components/HistoryExport/ToRemoteFile.test.js new file mode 100644 index 000000000000..275d2076bab6 --- /dev/null +++ b/client/src/components/HistoryExport/ToRemoteFile.test.js @@ -0,0 +1,72 @@ +import { shallowMount } from "@vue/test-utils"; +import { getLocalVue } from "jest/helpers"; +import ToRemoteFile from "./ToRemoteFile.vue"; +import MockAdapter from "axios-mock-adapter"; +import axios from "axios"; +import flushPromises from "flush-promises"; +import { waitOnJob } from "components/JobStates/wait"; + +const localVue = getLocalVue(); +const TEST_HISTORY_ID = "hist1235"; +const TEST_JOB_ID = "job123789"; +const TEST_EXPORTS_URL = `/api/histories/${TEST_HISTORY_ID}/exports`; + +jest.mock("components/JobStates/wait"); + +describe("ToRemoteFile.vue", () => { + let axiosMock; + let wrapper; + + beforeEach(async () => { + axiosMock = new MockAdapter(axios); + wrapper = shallowMount(ToRemoteFile, { + propsData: { + historyId: TEST_HISTORY_ID, + }, + localVue, + }); + }); + + it("should render a form with export disable because inputs empty", async () => { + expect(wrapper.find(".export-button").exists()).toBeTruthy(); + expect(wrapper.find(".export-button").attributes("disabled")).toBeTruthy(); + expect(wrapper.vm.canExport).toBeFalsy(); + }); + + it("should allow export when name and directory available", async () => { + await wrapper.setData({ + name: "export.tar.gz", + directory: "gxfiles://", + }); + expect(wrapper.vm.directory).toEqual("gxfiles://"); + expect(wrapper.vm.name).toEqual("export.tar.gz"); + expect(wrapper.vm.canExport).toBeTruthy(); + }); + + it("should issue export PUT request on export", async () => { + await wrapper.setData({ + name: "export.tar.gz", + directory: "gxfiles://", + }); + let request; + axiosMock.onPut(TEST_EXPORTS_URL).reply((request_) => { + request = request_; + return [200, { job_id: TEST_JOB_ID }]; + }); + waitOnJob.mockReturnValue( + new Promise((then) => { + then({ state: "ok" }); + }) + ); + wrapper.vm.doExport(); + await flushPromises(); + const putData = JSON.parse(request.data); + expect(putData.directory_uri).toEqual("gxfiles://"); + expect(putData.file_name).toEqual("export.tar.gz"); + expect(wrapper.find("b-alert-stub").attributes("variant")).toEqual("success"); + }); + + afterEach(() => { + axiosMock.restore(); + }); +}); diff --git a/client/src/components/HistoryExport/ToRemoteFile.vue b/client/src/components/HistoryExport/ToRemoteFile.vue new file mode 100644 index 000000000000..85ea86ad414d --- /dev/null +++ b/client/src/components/HistoryExport/ToRemoteFile.vue @@ -0,0 +1,119 @@ + + + diff --git a/client/src/components/HistoryExport/index.js b/client/src/components/HistoryExport/index.js new file mode 100644 index 000000000000..d1e4f95783fe --- /dev/null +++ b/client/src/components/HistoryExport/index.js @@ -0,0 +1 @@ +export { default as HistoryExport } from "./Index.vue"; diff --git a/client/src/components/HistoryImport.test.js b/client/src/components/HistoryImport.test.js new file mode 100644 index 000000000000..8c3c4bf20080 --- /dev/null +++ b/client/src/components/HistoryImport.test.js @@ -0,0 +1,79 @@ +import { shallowMount } from "@vue/test-utils"; +import { getLocalVue } from "jest/helpers"; +import HistoryImport from "./HistoryImport.vue"; +import MockAdapter from "axios-mock-adapter"; +import axios from "axios"; +import flushPromises from "flush-promises"; +import { waitOnJob } from "components/JobStates/wait"; + +const localVue = getLocalVue(); +const TEST_JOB_ID = "job123789"; +const TEST_HISTORY_URI = "/api/histories"; +const TEST_SOURCE_URL = "http://galaxy.example/import"; + +jest.mock("components/JobStates/wait"); + +describe("HistoryImport.vue", () => { + let axiosMock; + let wrapper; + + beforeEach(async () => { + axiosMock = new MockAdapter(axios); + wrapper = shallowMount(HistoryImport, { + propsData: {}, + localVue, + }); + }); + + it("should render a form with submit disabled because inputs empty", async () => { + expect(wrapper.find(".import-button").exists()).toBeTruthy(); + expect(wrapper.find(".import-button").attributes("disabled")).toBeTruthy(); + expect(wrapper.vm.importReady).toBeFalsy(); + }); + + it("should allow import when URL available", async () => { + await wrapper.setData({ + sourceURL: TEST_SOURCE_URL, + }); + expect(wrapper.vm.importReady).toBeTruthy(); + }); + + it("should require an URI if that is the import type", async () => { + await wrapper.setData({ + sourceURL: TEST_SOURCE_URL, + importType: "sourceRemoteFilesUri", + }); + expect(wrapper.vm.importReady).toBeFalsy(); + }); + + it("should post to create a new history and wait on job when submitted", async () => { + await wrapper.setData({ + sourceURL: TEST_SOURCE_URL, + }); + let formData; + axiosMock.onPost(TEST_HISTORY_URI).reply((request) => { + formData = request.data; + return [200, { job_id: TEST_JOB_ID }]; + }); + let then; + waitOnJob.mockReturnValue( + new Promise((then_) => { + then = then_; + }) + ); + wrapper.vm.submit(); + await flushPromises(); + expect(formData.get("archive_source")).toBe(TEST_SOURCE_URL); + expect(wrapper.vm.waitingOnJob).toBeTruthy(); + + // complete job and make sure waitingOnJob is false and complete is true + then({ state: "ok" }); + await flushPromises(); + expect(wrapper.vm.waitingOnJob).toBeFalsy(); + expect(wrapper.vm.complete).toBeTruthy(); + }); + + afterEach(() => { + axiosMock.restore(); + }); +}); diff --git a/client/src/components/HistoryImport.vue b/client/src/components/HistoryImport.vue index 5a3314bd1124..9b6a6d2069bd 100644 --- a/client/src/components/HistoryImport.vue +++ b/client/src/components/HistoryImport.vue @@ -1,57 +1,158 @@ + + diff --git a/client/src/components/JobInformation/JobError.vue b/client/src/components/JobInformation/JobError.vue new file mode 100644 index 000000000000..bab388d1f52c --- /dev/null +++ b/client/src/components/JobInformation/JobError.vue @@ -0,0 +1,70 @@ + + + + + diff --git a/client/src/components/JobInformation/JobInformation.vue b/client/src/components/JobInformation/JobInformation.vue index 792c495b107f..161621a3b5c6 100644 --- a/client/src/components/JobInformation/JobInformation.vue +++ b/client/src/components/JobInformation/JobInformation.vue @@ -11,6 +11,18 @@ Galaxy Tool Version: {{ job.tool_version }} + + Created + + + + + + Updated + + + + @@ -46,17 +58,23 @@ import { mapCacheActions } from "vuex-cache"; import { getAppRoot } from "onload/loadConfig"; import DecodedId from "../DecodedId.vue"; import CodeRow from "./CodeRow.vue"; +import UtcDate from "components/UtcDate"; export default { components: { CodeRow, DecodedId, + UtcDate, }, props: { job_id: { type: String, required: true, }, + includeTimes: { + type: Boolean, + default: false, + }, }, created: function () { this.fetchJob(this.job_id); diff --git a/client/src/components/JobStates/wait.js b/client/src/components/JobStates/wait.js new file mode 100644 index 000000000000..d942c7a16f81 --- /dev/null +++ b/client/src/components/JobStates/wait.js @@ -0,0 +1,29 @@ +import { getAppRoot } from "onload/loadConfig"; +import JOB_STATES_MODEL from "mvc/history/job-states-model"; +import axios from "axios"; + +export function waitOnJob(jobId, onStateUpdate = null, interval = 1000) { + const jobUrl = `${getAppRoot()}api/jobs/${jobId}`; + const checkCondition = function (resolve, reject) { + axios + .get(jobUrl) + .then((jobResponse) => { + const state = jobResponse.data.state; + if (onStateUpdate !== null) { + onStateUpdate(state); + } + if (JOB_STATES_MODEL.NON_TERMINAL_STATES.indexOf(state) !== -1) { + setTimeout(checkCondition, interval, resolve, reject); + } else if (JOB_STATES_MODEL.ERROR_STATES.indexOf(state) !== -1) { + // grab full output to include stderr and + // such when generating error messages. + axios.get(`${jobUrl}?full=true`).then(reject).catch(reject); + } else { + resolve(jobResponse); + } + }) + .catch(reject); + }; + + return new Promise(checkCondition); +} diff --git a/client/src/entry/analysis/AnalysisRouter.js b/client/src/entry/analysis/AnalysisRouter.js index 3cd82ff0e7d3..57b307aecb4c 100644 --- a/client/src/entry/analysis/AnalysisRouter.js +++ b/client/src/entry/analysis/AnalysisRouter.js @@ -32,6 +32,7 @@ import InteractiveTools from "components/InteractiveTools/InteractiveTools.vue"; import LibraryFolder from "components/LibraryFolder/LibraryFolder.vue"; import WorkflowList from "components/Workflow/WorkflowList.vue"; import HistoryImport from "components/HistoryImport.vue"; +import { HistoryExport } from "components/HistoryExport/index"; import HistoryView from "components/HistoryView.vue"; import WorkflowInvocationReport from "components/Workflow/InvocationReport.vue"; import WorkflowRun from "components/Workflow/Run/WorkflowRun.vue"; @@ -87,6 +88,7 @@ export const getAnalysisRouter = (Galaxy) => "(/)histories(/)rename(/)": "show_histories_rename", "(/)histories(/)sharing(/)": "show_histories_sharing", "(/)histories(/)import(/)": "show_histories_import", + "(/)histories(/)(:history_id)(/)export(/)": "show_history_export", "(/)histories(/)permissions(/)": "show_histories_permissions", "(/)histories/view": "show_history_view", "(/)histories/show_structure": "show_history_structure", @@ -251,6 +253,12 @@ export const getAnalysisRouter = (Galaxy) => this._display_vue_helper(HistoryImport); }, + show_history_export: function (history_id) { + this._display_vue_helper(HistoryExport, { + historyId: history_id, + }); + }, + show_tools_view: function () { this.page.toolPanel?.component.hide(); this.page.panels.right.hide(); diff --git a/client/src/mvc/history/job-states-model.js b/client/src/mvc/history/job-states-model.js index 06b745f51a33..4d7c4d04b75b 100644 --- a/client/src/mvc/history/job-states-model.js +++ b/client/src/mvc/history/job-states-model.js @@ -208,4 +208,10 @@ var JobStatesSummaryCollection = Backbone.Collection.extend({ }, }); -export default { JobStatesSummary, JobStatesSummaryCollection, FETCH_STATE_ON_ADD, NON_TERMINAL_STATES, ERROR_STATES }; +export default { + JobStatesSummary, + JobStatesSummaryCollection, + FETCH_STATE_ON_ADD, + NON_TERMINAL_STATES, + ERROR_STATES, +}; diff --git a/client/src/mvc/history/options-menu.js b/client/src/mvc/history/options-menu.js index 3192641c2f37..3556ca5b8dba 100644 --- a/client/src/mvc/history/options-menu.js +++ b/client/src/mvc/history/options-menu.js @@ -148,8 +148,13 @@ var menu = [ }, { html: _l("Export History to File"), - href: "history/export_archive?preview=True", anon: true, + func: function () { + const Galaxy = getGalaxyInstance(); + if (Galaxy && Galaxy.currHistoryPanel && Galaxy.router) { + Galaxy.router.push(`/histories/${Galaxy.currHistoryPanel.model.id}/export`); + } + }, }, { html: _l("Beta Features"), diff --git a/client/src/utils/simple-error.js b/client/src/utils/simple-error.js index aa5714b8a99a..75b7ce3bc18a 100644 --- a/client/src/utils/simple-error.js +++ b/client/src/utils/simple-error.js @@ -1,5 +1,5 @@ -export function errorMessageAsString(e) { - let message = "Request failed."; +export function errorMessageAsString(e, defaultMessage = "Request failed.") { + let message = defaultMessage; if (e && e.response && e.response.data && e.response.data.err_msg) { message = e.response.data.err_msg; } else if (e && e.response) { diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index dc27068c6959..f5ead2833715 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -169,6 +169,53 @@ def non_ready_jobs(self, history): return jobs +class HistoryExportView: + + def __init__(self, app): + self.app = app + + def get_exports(self, trans, history_id): + history = self._history(trans, history_id) + matching_exports = history.exports + return [self.serialize(trans, history_id, e) for e in matching_exports] + + def serialize(self, trans, history_id, jeha): + rval = jeha.to_dict() + encoded_jeha_id = trans.security.encode_id(jeha.id) + api_url = self.app.url_for("history_archive_download", id=history_id, jeha_id=encoded_jeha_id) + # this URL is less likely to be blocked by a proxy and require an API key, so export + # older-style controller version for use with within the GUI and such. + external_url = self.app.url_for(controller='history', action="export_archive", id=history_id, qualified=True) + external_permanent_url = self.app.url_for(controller='history', action="export_archive", id=history_id, jeha_id=encoded_jeha_id, qualified=True) + rval["download_url"] = api_url + rval["external_download_latest_url"] = external_url + rval["external_download_permanent_url"] = external_permanent_url + rval = trans.security.encode_all_ids(rval) + return rval + + def get_ready_jeha(self, trans, history_id, jeha_id="latest"): + history = self._history(trans, history_id) + matching_exports = history.exports + if jeha_id != "latest": + decoded_jeha_id = trans.security.decode_id(jeha_id) + matching_exports = [e for e in matching_exports if e.id == decoded_jeha_id] + if len(matching_exports) == 0: + raise glx_exceptions.ObjectNotFound("Failed to find target history export") + + jeha = matching_exports[0] + if not jeha.ready: + raise glx_exceptions.MessageException("Export not available or not yet ready.") + + return jeha + + def _history(self, trans, history_id): + if history_id is not None: + history = self.app.history_manager.get_accessible(trans.security.decode_id(history_id), trans.user, current_history=trans.history) + else: + history = trans.history + return history + + class HistorySerializer(sharable.SharableModelSerializer, deletable.PurgableSerializerMixin): """ Interface/service object for serializing histories into dictionaries. diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 65901a9d2691..d37b265b4e91 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1667,6 +1667,15 @@ def create_for_history(history, job, sa_session, object_store, compressed): jeha.history_attrs_filename = history_attrs_filename return jeha + def to_dict(self): + return { + 'id': self.id, + 'job_id': self.job.id, + 'ready': self.ready, + 'preparing': self.preparing, + 'up_to_date': self.up_to_date, + } + class JobImportHistoryArchive(RepresentById): def __init__(self, job=None, history=None, archive_dir=None): diff --git a/lib/galaxy/webapps/base/controller.py b/lib/galaxy/webapps/base/controller.py index 4fc92b96a093..07d49494552d 100644 --- a/lib/galaxy/webapps/base/controller.py +++ b/lib/galaxy/webapps/base/controller.py @@ -445,7 +445,8 @@ def queue_history_import(self, trans, archive_type, archive_source): # Run job to do import. history_imp_tool = trans.app.toolbox.get_tool('__IMPORT_HISTORY__') incoming = {'__ARCHIVE_SOURCE__' : archive_source, '__ARCHIVE_TYPE__' : archive_type} - history_imp_tool.execute(trans, incoming=incoming) + job, _ = history_imp_tool.execute(trans, incoming=incoming) + return job class UsesLibraryMixin: diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index 274a8d48f984..6a1e409148cb 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -32,7 +32,6 @@ expose_api_anonymous, expose_api_anonymous_and_sessionless, expose_api_raw, - url_for ) from galaxy.webapps.base.controller import ( BaseAPIController, @@ -52,6 +51,7 @@ def __init__(self, app): self.user_manager = users.UserManager(app) self.workflow_manager = workflows.WorkflowsManager(app) self.manager = histories.HistoryManager(app) + self.history_export_view = histories.HistoryExportView(app) self.serializer = histories.HistorySerializer(app) self.deserializer = histories.HistoryDeserializer(app) self.filters = histories.HistoryFilters(app) @@ -333,8 +333,10 @@ def create(self, trans, payload, **kwd): archive_type = "file" else: raise exceptions.MessageException("Please provide a url or file.") - self.queue_history_import(trans, archive_type=archive_type, archive_source=archive_source) - return {"message": "Importing history from source '%s'. This history will be visible when the import is complete." % archive_source} + job = self.queue_history_import(trans, archive_type=archive_type, archive_source=archive_source) + job_dict = job.to_dict() + job_dict["message"] = "Importing history from source '%s'. This history will be visible when the import is complete." % archive_source + return trans.security.encode_all_ids(job_dict) new_history = None # if a history id was passed, copy that history @@ -451,7 +453,16 @@ def update(self, trans, id, payload, **kwd): user=trans.user, trans=trans, **self._parse_serialization_params(kwd, 'detailed')) @expose_api - def archive_export(self, trans, id, **kwds): + def index_exports(self, trans, id): + """ + index_exports(self, trans, id) + * GET /api/histories/{id}/exports: + Get history exports. + """ + return self.history_export_view.get_exports(trans, id) + + @expose_api + def archive_export(self, trans, id, payload=None, **kwds): """ export_archive(self, trans, id, payload) * PUT /api/histories/{id}/exports: @@ -464,6 +475,7 @@ def archive_export(self, trans, id, **kwds): :rtype: dict :returns: object containing url to fetch export from. """ + kwds.update(payload or {}) # PUT instead of POST because multiple requests should just result # in one object being created. history = self.manager.get_accessible(self.decode_id(id), trans.user, current_history=trans.history) @@ -496,13 +508,16 @@ def archive_export(self, trans, id, **kwds): # written. job_id = trans.security.encode_id(job.id) return dict(job_id=job_id) + if up_to_date and jeha.ready: - jeha_id = trans.security.encode_id(jeha.id) - return dict(download_url=url_for("history_archive_download", id=id, jeha_id=jeha_id)) + return self.history_export_view.serialize(trans, id, jeha) else: # Valid request, just resource is not ready yet. trans.response.status = "202 Accepted" - return '' + if jeha: + return self.history_export_view.serialize(trans, id, jeha) + else: + return '' @expose_api_raw def archive_download(self, trans, id, jeha_id, **kwds): @@ -515,19 +530,7 @@ def archive_download(self, trans, id, jeha_id, **kwds): code (instead of 202) with a JSON dictionary containing a `download_url`. """ - # Seems silly to put jeha_id in here, but want GET to be immuatable? - # and this is being accomplished this way. - history = self.manager.get_accessible(self.decode_id(id), trans.user, current_history=trans.history) - matching_exports = [e for e in history.exports if trans.security.encode_id(e.id) == jeha_id] - if not matching_exports: - raise exceptions.ObjectNotFound() - - jeha = matching_exports[0] - if not jeha.ready: - # User should not have been given this URL, PUT export should have - # return a 202. - raise exceptions.MessageException("Export not available or not yet ready.") - + jeha = self.history_export_view.get_ready_jeha(trans, id, jeha_id) return self.serve_ready_history_export(trans, jeha) @expose_api diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index f94878736e61..225d37da7d8c 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -146,6 +146,7 @@ def app_factory(global_conf, load_app_kwds=None, **kwargs): webapp.add_client_route('/histories/citations') webapp.add_client_route('/histories/list') webapp.add_client_route('/histories/import') + webapp.add_client_route('/histories/{history_id}/export') webapp.add_client_route('/histories/list_published') webapp.add_client_route('/histories/list_shared') webapp.add_client_route('/histories/rename') @@ -496,6 +497,9 @@ def populate_api_routes(webapp, app): controller='page_revisions', parent_resources=dict(member_name='page', collection_name='pages')) + webapp.mapper.connect("history_exports", + "/api/histories/{id}/exports", controller="histories", + action="index_exports", conditions=dict(method=["GET"])) webapp.mapper.connect("history_archive_export", "/api/histories/{id}/exports", controller="histories", action="archive_export", conditions=dict(method=["PUT"])) diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py index 34225fd0be75..3d83ea8dfbd0 100644 --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -240,6 +240,7 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt def __init__(self, app): super().__init__(app) self.history_manager = managers.histories.HistoryManager(app) + self.history_export_view = managers.histories.HistoryExportView(app) self.history_serializer = managers.histories.HistorySerializer(self.app) @web.expose @@ -1097,39 +1098,13 @@ def rate_async(self, trans, id, rating): # TODO: used in display_base.mako @web.expose - def export_archive(self, trans, id=None, gzip=True, include_hidden=False, include_deleted=False, preview=False): + def export_archive(self, trans, id=None, jeha_id="latest"): """ Export a history to an archive. """ # # Get history to export. # - if id: - history = self.history_manager.get_accessible(self.decode_id(id), trans.user, current_history=trans.history) - else: - # Use current history. - history = trans.history - id = trans.security.encode_id(history.id) - if not history: - return trans.show_error_message("This history does not exist or you cannot export this history.") - # If history has already been exported and it has not changed since export, stream it. - jeha = history.latest_export - if jeha and jeha.up_to_date: - if jeha.ready: - if preview: - url = url_for(controller='history', action="export_archive", id=id, qualified=True) - return trans.show_message("History Ready: '%(n)s'. Use this link to download " - "the archive or import it to another Galaxy server: " - "%(u)s" % ({'n': history.name, 'u': url})) - else: - return self.serve_ready_history_export(trans, jeha) - elif jeha.preparing: - return trans.show_message("Still exporting history %(n)s; please check back soon. Link: %(s)s" - % ({'n': history.name, 's': url_for(controller='history', action="export_archive", id=id, qualified=True)})) - self.queue_history_export(trans, history, gzip=gzip, include_hidden=include_hidden, include_deleted=include_deleted) - url = url_for(controller='history', action="export_archive", id=id, qualified=True) - return trans.show_message("Exporting History '%(n)s'. You will need to make this history 'accessible' in order to import this to another galaxy sever.
" - "Use this link to download the archive or import it to another Galaxy server: " - "%(u)s" % ({'share': url_for('/histories/sharing', id=id), 'n': history.name, 'u': url})) - # TODO: used in this file and index.mako + jeha = self.history_export_view.get_ready_jeha(trans, id, jeha_id) + return self.serve_ready_history_export(trans, jeha) @web.expose @web.json From b1fc8430bb6b743954b2221d2141f016d0d1e4c9 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 8 Jan 2021 15:41:55 -0500 Subject: [PATCH 2/2] Revise history import/export based on Marius PR review. --- .../components/HistoryExport/Index.test.js | 15 ++++++ client/src/components/HistoryExport/Index.vue | 52 ++++++++++++++----- .../src/components/HistoryExport/ToLink.vue | 21 +++++--- client/src/components/HistoryImport.test.js | 3 ++ client/src/components/HistoryImport.vue | 18 ++++++- client/src/components/JobStates/wait.js | 8 +-- lib/galaxy/webapps/galaxy/api/histories.py | 6 ++- 7 files changed, 97 insertions(+), 26 deletions(-) diff --git a/client/src/components/HistoryExport/Index.test.js b/client/src/components/HistoryExport/Index.test.js index 6d133bbbdcb2..830104b24914 100644 --- a/client/src/components/HistoryExport/Index.test.js +++ b/client/src/components/HistoryExport/Index.test.js @@ -1,10 +1,21 @@ import { shallowMount } from "@vue/test-utils"; import Index from "./Index.vue"; import { getLocalVue } from "jest/helpers"; +import axios from "axios"; +import MockAdapter from "axios-mock-adapter"; + +const TEST_PLUGINS_URL = "/api/remote_files/plugins"; const localVue = getLocalVue(); describe("Index.vue", () => { + let axiosMock; + + beforeEach(() => { + axiosMock = new MockAdapter(axios); + axiosMock.onGet(TEST_PLUGINS_URL).reply(200, [{ id: "foo", writable: false }]); + }); + it("should render tabs", () => { // just make sure the component renders to catch obvious big errors const wrapper = shallowMount(Index, { @@ -15,4 +26,8 @@ describe("Index.vue", () => { }); expect(wrapper.exists("b-tabs-stub")).toBeTruthy(); }); + + afterEach(() => { + axiosMock.restore(); + }); }); diff --git a/client/src/components/HistoryExport/Index.vue b/client/src/components/HistoryExport/Index.vue index 3c9f2bb1e503..51db0e2fccda 100644 --- a/client/src/components/HistoryExport/Index.vue +++ b/client/src/components/HistoryExport/Index.vue @@ -1,18 +1,26 @@ @@ -21,11 +29,14 @@ import Vue from "vue"; import BootstrapVue from "bootstrap-vue"; import ToLink from "./ToLink.vue"; import ToRemoteFile from "./ToRemoteFile.vue"; +import { Services } from "components/FilesDialog/services"; +import LoadingSpan from "components/LoadingSpan"; Vue.use(BootstrapVue); export default { components: { + LoadingSpan, ToLink, ToRemoteFile, }, @@ -35,5 +46,22 @@ export default { required: true, }, }, + data() { + return { + initializing: true, + hasWritableFileSources: false, + }; + }, + async mounted() { + await this.initialize(); + }, + methods: { + async initialize() { + const fileSources = await new Services().getFileSources(); + this.hasWritableFileSources = fileSources.some((fs) => fs.writable); + console.log(fileSources); + this.initializing = false; + }, + }, }; diff --git a/client/src/components/HistoryExport/ToLink.vue b/client/src/components/HistoryExport/ToLink.vue index 8cb5f8fc036d..b45587bac269 100644 --- a/client/src/components/HistoryExport/ToLink.vue +++ b/client/src/components/HistoryExport/ToLink.vue @@ -25,7 +25,7 @@

The history has changed since this export was generated, - click here to generate a new archive for the current history state.

@@ -34,7 +34,7 @@

No link for history export ready, {{ whyNoLink }}.

Click here to to generate a new archive for this history. @@ -67,18 +67,20 @@ export default { data() { return { errorMessage: null, + exportsInitialized: false, exports: null, + loadingExports: false, waitingOnJob: false, jobError: null, }; }, created() { - this.loadExports(); + if (!this.exportsInitialized) { + this.exportsInitialized = true; + this.loadExports(); + } }, computed: { - loadingExports() { - return this.exports === null; - }, hasExports() { return this.exports !== null && this.exports.length > 0; }, @@ -110,10 +112,12 @@ export default { }, methods: { loadExports() { + this.loadingExports = true; const url = `${getAppRoot()}api/histories/${this.historyId}/exports`; axios .get(url) .then((response) => { + this.loadingExports = false; this.exports = response.data; if (this.latestExportPreparing) { this.waitOnExportJob(this.latestExport.job_id); @@ -147,7 +151,9 @@ export default { this.waitingOnJob = true; this.jobError = false; waitOnJob(jobId) - .then((data) => { + .then((jobResponse) => { + this.waitingOnJob = false; + this.loadingExports = true; // Race condition, for some reasons the job API returns // ok before the JEHA... setTimeout(this.loadExports, 2000); @@ -156,6 +162,7 @@ export default { }, handleError(err) { this.waitingOnJob = false; + this.loadingExports = false; this.errorMessage = errorMessageAsString(err, "Error generating a history export link"); if (err?.data?.stderr) { this.jobError = err.data; diff --git a/client/src/components/HistoryImport.test.js b/client/src/components/HistoryImport.test.js index 8c3c4bf20080..2e304a50588e 100644 --- a/client/src/components/HistoryImport.test.js +++ b/client/src/components/HistoryImport.test.js @@ -10,6 +10,7 @@ const localVue = getLocalVue(); const TEST_JOB_ID = "job123789"; const TEST_HISTORY_URI = "/api/histories"; const TEST_SOURCE_URL = "http://galaxy.example/import"; +const TEST_PLUGINS_URL = "/api/remote_files/plugins"; jest.mock("components/JobStates/wait"); @@ -19,10 +20,12 @@ describe("HistoryImport.vue", () => { beforeEach(async () => { axiosMock = new MockAdapter(axios); + axiosMock.onGet(TEST_PLUGINS_URL).reply(200, [{ id: "foo", writable: false }]); wrapper = shallowMount(HistoryImport, { propsData: {}, localVue, }); + await flushPromises(); }); it("should render a form with submit disabled because inputs empty", async () => { diff --git a/client/src/components/HistoryImport.vue b/client/src/components/HistoryImport.vue index 9b6a6d2069bd..305000896a0f 100644 --- a/client/src/components/HistoryImport.vue +++ b/client/src/components/HistoryImport.vue @@ -9,7 +9,10 @@ :job="jobError" /> -

+
+ +
+
@@ -35,7 +38,7 @@ Upload local file from your computer - + Select a remote file (e.g. Galaxy's FTP) @@ -71,6 +74,7 @@ import { waitOnJob } from "components/JobStates/wait"; import { errorMessageAsString } from "utils/simple-error"; import LoadingSpan from "components/LoadingSpan"; import JobError from "components/JobInformation/JobError"; +import { Services } from "components/FilesDialog/services"; library.add(faFolderOpen); library.add(faUpload); @@ -81,6 +85,7 @@ export default { components: { FilesInput, FontAwesomeIcon, JobError, LoadingSpan }, data() { return { + initializing: true, importType: "externalUrl", sourceFile: null, sourceURL: null, @@ -89,8 +94,12 @@ export default { waitingOnJob: false, complete: false, jobError: null, + hasFileSources: false, }; }, + async mounted() { + await this.initialize(); + }, computed: { importReady() { const importType = this.importType; @@ -109,6 +118,11 @@ export default { }, }, methods: { + async initialize() { + const fileSources = await new Services().getFileSources(); + this.hasFileSources = fileSources.length > 0; + this.initializing = false; + }, submit: function (ev) { const formData = new FormData(); const importType = this.importType; diff --git a/client/src/components/JobStates/wait.js b/client/src/components/JobStates/wait.js index d942c7a16f81..880edd3afffd 100644 --- a/client/src/components/JobStates/wait.js +++ b/client/src/components/JobStates/wait.js @@ -3,7 +3,9 @@ import JOB_STATES_MODEL from "mvc/history/job-states-model"; import axios from "axios"; export function waitOnJob(jobId, onStateUpdate = null, interval = 1000) { - const jobUrl = `${getAppRoot()}api/jobs/${jobId}`; + // full=true to capture standard error on last iteration for building + // error messages. + const jobUrl = `${getAppRoot()}api/jobs/${jobId}?full=true`; const checkCondition = function (resolve, reject) { axios .get(jobUrl) @@ -15,9 +17,7 @@ export function waitOnJob(jobId, onStateUpdate = null, interval = 1000) { if (JOB_STATES_MODEL.NON_TERMINAL_STATES.indexOf(state) !== -1) { setTimeout(checkCondition, interval, resolve, reject); } else if (JOB_STATES_MODEL.ERROR_STATES.indexOf(state) !== -1) { - // grab full output to include stderr and - // such when generating error messages. - axios.get(`${jobUrl}?full=true`).then(reject).catch(reject); + reject(jobResponse); } else { resolve(jobResponse); } diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index 6a1e409148cb..f5a6481eec2e 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -501,6 +501,8 @@ def archive_export(self, trans, id, payload=None, **kwds): directory_uri=directory_uri, file_name=file_name, ) + else: + job = jeha.job if exporting_to_uri: # we don't have a jeha, there will never be a download_url. Just let @@ -517,7 +519,9 @@ def archive_export(self, trans, id, payload=None, **kwds): if jeha: return self.history_export_view.serialize(trans, id, jeha) else: - return '' + assert job is not None, "logic error, don't have a jeha or a job" + job_id = trans.security.encode_id(job.id) + return dict(job_id=job_id) @expose_api_raw def archive_download(self, trans, id, jeha_id, **kwds):