User-facing objectstore metadata.

client/src/bundleEntries.js

@@ -96,6 +96,7 @@ export { mountWorkflowEditor } from "components/Workflow/Editor/mount";
 export { mountPageEditor } from "components/PageEditor/mount";
 export { mountPageDisplay } from "components/PageDisplay";
 export { mountDestinationParams } from "components/JobDestinationParams";
+export { mountDatasetStorage } from "components/Dataset/DatasetStorage";
 
 // Used in common.mako
 export { default as store } from "storemodern";

client/src/components/Dataset/DatasetStorage/DatasetStorage.vue

@@ -0,0 +1,69 @@
+<template>
+    <div>
+        <h3 v-if="includeTitle">Dataset Storage</h3>
+        <loading-span v-if="storageInfo == null"> </loading-span>
+        <div v-else>
+            <p v-if="storageInfo.name">
+                This data is stored in a Galaxy ObjectStore named <b>{{ storageInfo.name }}</b
+                >.
+            </p>
+            <p v-if="storageInfo.object_store_id">
+                This data is stored in a Galaxy ObjectStore with id <b>{{ storageInfo.object_store_id }}</b
+                >.
+            </p>
+            <div v-html="descriptionRendered"></div>
+        </div>
+    </div>
+</template>
+
+<script>
+import axios from "axios";
+import { getAppRoot } from "onload/loadConfig";
+import LoadingSpan from "components/LoadingSpan";
+import MarkdownIt from "markdown-it";
+
+const md = MarkdownIt();
+
+export default {
+    components: {
+        LoadingSpan,
+    },
+    props: {
+        datasetId: {
+            type: String,
+        },
+        datasetType: {
+            type: String,
+            default: "hda",
+        },
+        includeTitle: {
+            type: Boolean,
+            default: true,
+        },
+    },
+    data() {
+        return {
+            storageInfo: null,
+            descriptionRendered: null,
+        };
+    },
+    created() {
+        const datasetId = this.datasetId;
+        const datasetType = this.datasetType;
+        axios
+            .get(`${getAppRoot()}api/datasets/${datasetId}/storage?hda_ldda=${datasetType}`)
+            .then(this.handleResponse)
+            .catch(this.handleError);
+    },
+    methods: {
+        handleResponse(response) {
+            console.log(response);
+            this.storageInfo = response.data;
+            this.descriptionRendered = md.render(this.storageInfo.description);
+        },
+        handleError(err) {
+            console.log(err);
+        },
+    },
+};
+</script>

client/src/components/Dataset/DatasetStorage/index.js

@@ -0,0 +1,4 @@
+export { default as DatasetStorage } from "./DatasetStorage";
+
+// functions for mounting job metrics in non-Vue environments
+export { mountDatasetStorage } from "./mount";

client/src/components/Dataset/DatasetStorage/mount.js

@@ -0,0 +1,16 @@
+/**
+ * Endpoint for mounting job metrics from non-Vue environment.
+ */
+import DatasetStorage from "./DatasetStorage.vue";
+import { mountVueComponent } from "utils/mountVueComponent";
+
+export const mountDatasetStorage = (propsData = {}) => {
+    const elements = document.querySelectorAll(".dataset-storage");
+    Array.prototype.forEach.call(elements, function (el, i) {
+        const datasetId = el.getAttribute("dataset_id");
+        const datasetType = el.getAttribute("dataset_type") || "hda";
+        propsData.datasetId = datasetId;
+        propsData.datasetType = datasetType;
+        mountVueComponent(DatasetStorage)(propsData, el);
+    });
+};

lib/galaxy/objectstore/__init__.py

@@ -171,12 +171,42 @@ def update_from_file(self, obj, base_dir=None, extra_dir=None, extra_dir_at_root
     @abc.abstractmethod
     def get_object_url(self, obj, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False):
         """
-        Return the URL for direct acces if supported, otherwise return None.
+        Return the URL for direct access if supported, otherwise return None.
 
         Note: need to be careful to not bypass dataset security with this.
         """
         raise NotImplementedError()
 
+    @abc.abstractmethod
+    def get_concrete_store_name(self, obj):
+        """Return a display name or title of the objectstore corresponding to obj.
+
+        To accommodate nested objectstores, obj is passed in so this metadata can
+        be returned for the ConcreteObjectStore corresponding to the object.
+        """
+
+    @abc.abstractmethod
+    def get_concrete_store_description_markdown(self, obj):
+        """Return a longer description of how data 'obj' is stored.
+
+        To accommodate nested objectstores, obj is passed in so this metadata can
+        be returned for the ConcreteObjectStore corresponding to the object.
+        """
+
+    @abc.abstractmethod
+    def is_transient(self, obj):
+        """Return boolean indicating if obj should be treated as transient.
+
+        To accommodate nested objectstores, obj is passed in so this metadata can
+        be returned for the ConcreteObjectStore corresponding to the object.
+
+        ObjectStores corresponding to data sources that get purged frequently
+        can mark that here. In the future it would be nice to provide different
+        icons in the UI for instance based on this or warn people before
+        publishing pages with links to transient datasets for instance, this
+        should provide a piece of the puzzle for doing that in the future.
+        """
+
     @abc.abstractmethod
     def get_store_usage_percent(self):
         """Return the percentage indicating how full the store is."""
@@ -292,6 +322,15 @@ def update_from_file(self, obj, **kwargs):
     def get_object_url(self, obj, **kwargs):
         return self._invoke('get_object_url', obj, **kwargs)
 
+    def get_concrete_store_name(self, obj):
+        return self._invoke('get_concrete_store_name', obj)
+
+    def get_concrete_store_description_markdown(self, obj):
+        return self._invoke('get_concrete_store_description_markdown', obj)
+
+    def is_transient(self, obj):
+        return self._invoke('is_transient', obj)
+
     def get_store_usage_percent(self):
         return self._invoke('get_store_usage_percent')
 
@@ -323,12 +362,27 @@ def __init__(self, config, config_dict=None, **kwargs):
             config_dict = {}
         super().__init__(config=config, config_dict=config_dict, **kwargs)
         self.store_by = config_dict.get("store_by", None) or getattr(config, "object_store_store_by", "id")
+        self.name = config_dict.get("name", None)
+        self.description = config_dict.get("description", None)
+        self.transient = config_dict.get("transient", False)
 
     def to_dict(self):
         rval = super().to_dict()
         rval["store_by"] = self.store_by
+        rval["name"] = self.name
+        rval["description"] = self.description
+        rval["transient"] = self.transient
         return rval
 
+    def _get_concrete_store_name(self, obj):
+        return self.name
+
+    def _get_concrete_store_description_markdown(self, obj):
+        return self.description
+
+    def _is_transient(self, obj):
+        return self.transient
+
     def _get_store_by(self, obj):
         return self.store_by
 
@@ -378,9 +432,17 @@ def parse_xml(clazz, config_xml):
             store_by = config_xml.attrib.get('store_by', None)
             if store_by is not None:
                 config_dict['store_by'] = store_by
+            transient = config_xml.attrib.get('transient', None)
+            if transient is not None:
+                config_dict['transient'] = asbool(transient)
+            name = config_xml.attrib.get('name', None)
+            if name is not None:
+                config_dict['name'] = name
             for e in config_xml:
                 if e.tag == 'files_dir':
                     config_dict["files_dir"] = e.get('path')
+                elif e.tag == 'description':
+                    config_dict["description"] = e.text
                 else:
                     extra_dirs.append({"type": e.get('type'), "path": e.get('path')})
 
@@ -659,6 +721,15 @@ def _get_object_url(self, obj, **kwargs):
         """For the first backend that has this `obj`, get its URL."""
         return self._call_method('_get_object_url', obj, None, False, **kwargs)
 
+    def _get_concrete_store_name(self, obj):
+        return self._call_method('_get_concrete_store_name', obj, None, True)
+
+    def _get_concrete_store_description_markdown(self, obj):
+        return self._call_method('_get_concrete_store_description_markdown', obj, None, True)
+
+    def _is_transient(self, obj):
+        return self._call_method('_is_transient', obj, None, True)
+
     def _get_store_by(self, obj):
         return self._call_method('_get_store_by', obj, None, False)
 

lib/galaxy/webapps/galaxy/api/datasets.py

@@ -155,6 +155,36 @@ def show(self, trans, id, hda_ldda='hda', data_type=None, provider=None, **kwd):
                 rval = dataset.to_dict()
         return rval
 
+    @web.expose_api_anonymous
+    def show_storage(self, trans, dataset_id, hda_ldda='hda', **kwd):
+        """
+        GET /api/datasets/{encoded_dataset_id}/storage
+
+        Display user-facing storage details related to the objectstore a
+        dataset resides in.
+        """
+        dataset_instance = self.get_hda_or_ldda(trans, hda_ldda=hda_ldda, dataset_id=dataset_id)
+        dataset = dataset_instance.dataset
+        object_store = self.app.object_store
+        object_store_id = dataset.object_store_id
+        name = object_store.get_concrete_store_name(dataset)
+        description = object_store.get_concrete_store_description_markdown(dataset)
+        transient = object_store.is_transient(dataset)
+        # not really working (existing problem)
+        try:
+            percent_used = object_store.get_store_usage_percent()
+        except AttributeError:
+            # not implemented on nestedobjectstores yet.
+            percent_used = None
+
+        return {
+            'object_store_id': object_store_id,
+            'name': name,
+            'description': description,
+            'transient': transient,
+            'percent_used': percent_used,
+        }
+
     @web.expose_api
     def update_permissions(self, trans, dataset_id, payload, **kwd):
         """

lib/galaxy/webapps/galaxy/buildapp.py

@@ -264,6 +264,11 @@ def populate_api_routes(webapp, app):
                           controller="datasets",
                           action="get_content_as_text",
                           conditions=dict(method=["GET"]))
+    webapp.mapper.connect('dataset_storage',
+                          '/api/datasets/{dataset_id}/storage',
+                          controller='datasets',
+                          action='show_storage',
+                          conditions=dict(method=["GET"]))
     webapp.mapper.connect("history_contents_metadata_file",
                           "/api/histories/{history_id}/contents/{history_content_id}/metadata_file",
                           controller="datasets",

lib/galaxy_test/base/populators.py

@@ -513,6 +513,11 @@ def ds_entry(self, history_content):
             src = 'hdca'
         return dict(src=src, id=history_content["id"])
 
+    def dataset_storage_info(self, dataset_id):
+        storage_response = self.galaxy_interactor.get("datasets/{}/storage".format(dataset_id))
+        storage_response.raise_for_status()
+        return storage_response.json()
+
     def get_roles(self):
         roles_response = self.galaxy_interactor.get("roles", admin=True)
         assert roles_response.status_code == 200

templates/show_params.mako

@@ -95,6 +95,9 @@
     </tbody>
 </table>
 
+<div class="dataset-storage" dataset_id="${encoded_hda_id}" dataset_type="hda">
+</div>
+
 %if job:
 <div class="job-parameters" dataset_id="${encoded_hda_id}" dataset_type="hda">
 </div>
@@ -184,5 +187,6 @@ $(function(){
     window.bundleEntries.mountJobMetrics();
     window.bundleEntries.mountJobParameters();
     window.bundleEntries.mountDestinationParams();
+    window.bundleEntries.mountDatasetStorage();
 });
 </script>

test/integration/objectstore/test_selection.py

@@ -12,7 +12,8 @@
 DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE = string.Template("""<?xml version="1.0"?>
 <object_store type="distributed" id="primary" order="0">
     <backends>
-        <backend id="default" type="disk" weight="1">
+        <backend id="default" type="disk" weight="1" name="Default Store">
+            <description>This is my description of the default store with *markdown*.</description>
             <files_dir path="${temp_directory}/files_default"/>
             <extra_dir type="temp" path="${temp_directory}/tmp_default"/>
             <extra_dir type="job_work" path="${temp_directory}/job_working_directory_default"/>
@@ -86,6 +87,9 @@ def _run_tool(tool_id, inputs):
             hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3")
             self.dataset_populator.wait_for_history(history_id)
             hda1_input = {"src": "hda", "id": hda1["id"]}
+            storage_info = self.dataset_populator.dataset_storage_info(hda1["id"])
+            assert "Default Store" == storage_info["name"]
+            assert "*markdown*" in storage_info["description"]
 
             # One file uploaded, added to default object store ID.
             self._assert_file_counts(1, 0, 0, 0)

test/unit/objectstore/test_objectstore.py

@@ -197,12 +197,20 @@ def test_disk_store_alt_name_abspath():
 HIERARCHICAL_TEST_CONFIG = """<?xml version="1.0"?>
 <object_store type="hierarchical">
     <backends>
-        <backend id="files1" type="disk" weight="1" order="0">
+        <backend id="files1" type="disk" weight="1" order="0" name="Newer Cool Storage">
+            <description>
+              This is our new storage cluster, check out the storage
+              on our institute's system page for [Fancy New Storage](http://computecenter.example.com/systems/fancystorage).
+            </description>
             <files_dir path="${temp_directory}/files1"/>
             <extra_dir type="temp" path="${temp_directory}/tmp1"/>
             <extra_dir type="job_work" path="${temp_directory}/job_working_directory1"/>
         </backend>
-        <backend id="files2" type="disk" weight="1" order="1">
+        <backend id="files2" type="disk" weight="1" order="1" name="Older Legacy Storage">
+            <description>
+              This is our older legacy storage cluster, check out the storage
+              on our institute's system page for [Legacy Storage](http://computecenter.example.com/systems/legacystorage).
+            </description>
             <files_dir path="${temp_directory}/files2"/>
             <extra_dir type="temp" path="${temp_directory}/tmp2"/>
             <extra_dir type="job_work" path="${temp_directory}/job_working_directory2"/>
@@ -215,6 +223,10 @@ def test_disk_store_alt_name_abspath():
 type: hierarchical
 backends:
    - id: files1
+     name: Newer Cool Storage
+     description: |
+      This is our new storage cluster, check out the storage
+      on our institute's system page for [Fancy New Storage](http://computecenter.example.com/systems/fancystorage).
      type: disk
      weight: 1
      files_dir: "${temp_directory}/files1"
@@ -224,6 +236,10 @@ def test_disk_store_alt_name_abspath():
      - type: job_work
        path: "${temp_directory}/job_working_directory1"
    - id: files2
+     name: Older Legacy Storage
+     description: |
+      This is our older legacy storage cluster, check out the storage
+      on our institute's system page for [Legacy Storage](http://computecenter.example.com/systems/legacystorage).
      type: disk
      weight: 1
      files_dir: "${temp_directory}/files2"
@@ -248,11 +264,21 @@ def test_hierarchical_store():
             assert object_store.exists(MockDataset(2))
             assert object_store.empty(MockDataset(2))
 
-            # Write non-empty dataset in backend 1, test it is not emtpy & exists.
+            # Write non-empty dataset in backend 1, test it is not empty & exists.
             directory.write("Hello World!", "files1/000/dataset_3.dat")
             assert object_store.exists(MockDataset(3))
             assert not object_store.empty(MockDataset(3))
 
+            # check and description routed correctly
+            files1_desc = object_store.get_concrete_store_description_markdown(MockDataset(3))
+            files1_name = object_store.get_concrete_store_name(MockDataset(3))
+            files2_desc = object_store.get_concrete_store_description_markdown(MockDataset(2))
+            files2_name = object_store.get_concrete_store_name(MockDataset(2))
+            assert "fancy" in files1_desc
+            assert "Newer Cool" in files1_name
+            assert "older" in files2_desc
+            assert "Legacy" in files2_name
+
             # Assert creation always happens in first backend.
             for i in range(100):
                 dataset = MockDataset(100 + i)