Merge pull request #10152 from jmchilton/galaxy_files_write

Infrastructure for writing to pluggable Galaxy file sources.

client/src/components/FilesDialog/FilesDialog.vue

@@ -60,6 +60,10 @@ export default {
             default: "file",
             validator: (prop) => ["file", "directory"].includes(prop),
         },
+        requireWritable: {
+            type: Boolean,
+            default: false,
+        },
     },
     data() {
         return {
@@ -129,16 +133,19 @@ export default {
                 this.services
                     .getFileSources()
                     .then((items) => {
-                        this.items = items.map((item) => {
-                            return {
-                                id: item.id,
-                                label: item.label,
-                                details: item.doc,
-                                isLeaf: false,
-                                url: item.uri_root,
-                                labelTitle: item.uri_root,
-                            };
-                        });
+                        items = items
+                            .filter((item) => !this.requireWritable || item.writable)
+                            .map((item) => {
+                                return {
+                                    id: item.id,
+                                    label: item.label,
+                                    details: item.doc,
+                                    isLeaf: false,
+                                    url: item.uri_root,
+                                    labelTitle: item.uri_root,
+                                };
+                            });
+                        this.items = items;
                         this.formatRows();
                         this.optionsShow = true;
                         this.showTime = false;

client/src/mvc/form/form-parameters.js

@@ -10,6 +10,7 @@ import SelectContent from "mvc/ui/ui-select-content";
 import SelectLibrary from "mvc/ui/ui-select-library";
 import SelectFtp from "mvc/ui/ui-select-ftp";
 import RulesEdit from "mvc/ui/ui-rules-edit";
+import FileSource from "mvc/ui/ui-file-source";
 import ColorPicker from "mvc/ui/ui-color-picker";
 import DataPicker from "mvc/ui/ui-data-picker";
 
@@ -37,6 +38,7 @@ export default Backbone.Model.extend({
         ftpfile: "_fieldFtp",
         upload: "_fieldUpload",
         rules: "_fieldRulesEdit",
+        directory_uri: "_fieldDirectoryUri",
         data_dialog: "_fieldDialog",
     },
 
@@ -238,6 +240,14 @@ export default Backbone.Model.extend({
         });
     },
 
+    _fieldDirectoryUri: function (input_def) {
+        return new FileSource.View({
+            id: `field-${input_def.id}`,
+            onchange: input_def.onchange,
+            target: input_def.target,
+        });
+    },
+
     _fieldRulesEdit: function (input_def) {
         return new RulesEdit.View({
             id: `field-${input_def.id}`,

client/src/mvc/ui/ui-file-source.js

@@ -0,0 +1,100 @@
+import Backbone from "backbone";
+import { filesDialog } from "utils/data";
+import _l from "utils/localization";
+import Ui from "mvc/ui/ui-misc";
+
+var View = Backbone.View.extend({
+    initialize: function (options) {
+        this.model = new Backbone.Model();
+        this.target = options.target;
+        const props = {
+            mode: "directory",
+            requireWritable: true,
+        };
+        // create insert edit button
+        this.browse_button = new Ui.Button({
+            title: _l("Select"),
+            icon: "fa fa-edit",
+            tooltip: _l("Select URI"),
+            onclick: () => {
+                filesDialog((uri) => {
+                    this._handleRemoteFilesUri(uri);
+                }, props);
+            },
+        });
+
+        // add change event. fires on trigger
+        this.on("change", () => {
+            if (options.onchange) {
+                options.onchange(this.value());
+            }
+        });
+
+        // create elements
+        this.setElement(this._template(options));
+        this.$text = this.$(".ui-uri-preview");
+        this.$(".ui-file-select-button").append(this.browse_button.$el);
+        this.listenTo(this.model, "change", this.render, this);
+        this.render();
+    },
+
+    _handleRemoteFilesUri: function (uri) {
+        this._setValue(uri);
+    },
+
+    render: function () {
+        const value = this._value;
+        if (value) {
+            if (value.url !== this.$text.text()) {
+                this.$text.text(value.url);
+            }
+        } else {
+            this.$text.text("select...");
+        }
+    },
+
+    /** Main Template */
+    _template: function (options) {
+        return `
+            <div class="ui-rules-edit clearfix">
+                <span class="ui-uri-preview" />
+                <span class="ui-file-select-button float-left" />
+            </div>
+        `;
+    },
+
+    /** Return/Set current value */
+    value: function (new_value) {
+        if (new_value !== undefined) {
+            this._setValue(new_value);
+        } else {
+            return this._getValue();
+        }
+    },
+
+    /** Update input element options */
+    update: function (input_def) {
+        this.target = input_def.target;
+    },
+
+    /** Returns current value */
+    _getValue: function () {
+        return this._value.url;
+    },
+
+    /** Sets current value */
+    _setValue: function (new_value) {
+        if (new_value) {
+            if (typeof new_value == "string") {
+                new_value = JSON.parse(new_value);
+            }
+            this._value = new_value;
+            this.model.trigger("error", null);
+            this.model.trigger("change");
+        }
+    },
+});
+
+export default {
+    View: View,
+};

lib/galaxy/files/sources/__init__.py

@@ -6,6 +6,9 @@
 
 from galaxy.util.template import fill_template
 
+DEFAULT_SCHEME = "gxfiles"
+DEFAULT_WRITABLE = False
+
 
 @six.add_metaclass(abc.ABCMeta)
 class FilesSource(object):
@@ -20,6 +23,10 @@ def get_uri_root(self):
     def get_scheme(self):
         """Return a prefix for the root (e.g. the gxfiles in gxfiles://prefix/path)."""
 
+    @abc.abstractmethod
+    def get_writable(self):
+        """Return a boolean indicating if this target is writable."""
+
     # TODO: off-by-default
     @abc.abstractmethod
     def list(self, source_path="/", recursive=False, user_context=None):
@@ -29,6 +36,10 @@ def list(self, source_path="/", recursive=False, user_context=None):
     def realize_to(self, source_path, native_path, user_context=None):
         """Realize source path (relative to uri root) to local file system path."""
 
+    def write_from(self, target_path, native_path, user_context=None):
+        """Write file at native path to target_path (relative to uri root).
+        """
+
     @abc.abstractmethod
     def to_dict(self, for_serialization=False, user_context=None):
         """Return a dictified representation of this FileSource instance.
@@ -46,6 +57,9 @@ def get_prefix(self):
     def get_scheme(self):
         return "gxfiles"
 
+    def get_writable(self):
+        return self.writable
+
     def get_uri_root(self):
         prefix = self.get_prefix()
         scheme = self.get_scheme()
@@ -63,7 +77,8 @@ def _parse_common_config_opts(self, kwd):
         self.id = kwd.pop("id")
         self.label = kwd.pop("label", None) or self.id
         self.doc = kwd.pop("doc", None)
-        self.scheme = kwd.pop("scheme", "gxfiles")
+        self.scheme = kwd.pop("scheme", DEFAULT_SCHEME)
+        self.writable = kwd.pop("writable", DEFAULT_WRITABLE)
         # If coming from to_dict, strip API helper values
         kwd.pop("uri_root", None)
         kwd.pop("type", None)
@@ -76,6 +91,7 @@ def to_dict(self, for_serialization=False, user_context=None):
             "uri_root": self.get_uri_root(),
             "label": self.label,
             "doc": self.doc,
+            "writable": self.writable,
         }
         if for_serialization:
             rval.update(self._serialization_props(user_context=user_context))
@@ -96,6 +112,15 @@ def _serialization_props(self):
         Used in to_dict method if for_serialization is True.
         """
 
+    def write_from(self, target_path, native_path, user_context=None):
+        if not self.get_writable():
+            raise Exception("Cannot write to a non-writable file source.")
+        self._write_from(target_path, native_path, user_context=user_context)
+
+    @abc.abstractmethod
+    def _write_from(self, target_path, native_path, user_context=None):
+        pass
+
     def _evaluate_prop(self, prop_val, user_context):
         rval = prop_val
         if isinstance(prop_val, str) and "$" in prop_val:

lib/galaxy/files/sources/_pyfilesystem2.py

@@ -42,6 +42,10 @@ def realize_to(self, source_path, native_path, user_context=None):
         with open(native_path, 'wb') as write_file:
             self._open_fs(user_context=user_context).download(source_path, write_file)
 
+    def _write_from(self, target_path, native_path, user_context=None):
+        with open(native_path, 'rb') as read_file:
+            self._open_fs(user_context=user_context).upload(target_path, read_file)
+
     def _resource_info_to_dict(self, dir_path, resource_info):
         name = resource_info.name
         path = os.path.join(dir_path, name)

lib/galaxy/files/sources/galaxy.py

@@ -12,6 +12,7 @@ def __init__(self, label="FTP Directory", doc="Galaxy User's FTP Directory", roo
             root=root,
             label=label,
             doc=doc,
+            writable=True,
         )
         posix_kwds.update(kwd)
         if "delete_on_realize" not in posix_kwds:

lib/galaxy/files/sources/posix.py

@@ -62,6 +62,22 @@ def realize_to(self, source_path, native_path, user_context=None):
         else:
             shutil.move(source_native_path, native_path)
 
+    def _write_from(self, target_path, native_path, user_context=None):
+        effective_root = self._effective_root(user_context)
+        target_native_path = self._to_native_path(target_path, user_context=user_context)
+        if self.enforce_symlink_security:
+            if not safe_contains(effective_root, target_native_path, allowlist=self._allowlist):
+                raise Exception("Operation not allowed.")
+        else:
+            target_native_path = os.path.normpath(target_native_path)
+            assert target_native_path.startswith(os.path.normpath(effective_root))
+
+        target_native_path_parent = os.path.dirname(target_native_path)
+        if not os.path.exists(target_native_path_parent):
+            raise Exception("Parent directory does not exist.")
+
+        shutil.copyfile(native_path, target_native_path)
+
     def _to_native_path(self, source_path, user_context=None):
         source_path = os.path.normpath(source_path)
         if source_path.startswith("/"):

lib/galaxy/tool_util/xsd/galaxy.xsd

@@ -2398,6 +2398,14 @@ sanitizer to prevent Galaxy from escaping the result.
 </param>
 ```
 
+#### ``directory_uri``
+
+This is used to tie into galaxy.files URI infrastructure. This should only be used by
+core Galaxy tools until the interface around files has stabilized.
+
+Currently ``directory_uri`` parameters provide user's the option of selecting a writable
+directory destination for unstructured outputs of tools (e.g. history exports).
+
 $attribute_list:value,rgb:5
 
 This covers examples of the most common parameter types, the remaining parameter
@@ -2693,6 +2701,7 @@ allow access to Python code to generate options for a select list. See
       <xs:enumeration value="drill_down"/>
       <xs:enumeration value="group_tag"/>
       <xs:enumeration value="data_collection"/>
+      <xs:enumeration value="directory_uri" />
     </xs:restriction>
   </xs:simpleType>
 

lib/galaxy/tools/__init__.py

@@ -129,6 +129,7 @@
     "upload1",
     "send_to_cloud",
     "__DATA_FETCH__",
+    "directory_uri",
     # Legacy tools bundled with Galaxy.
     "laj_1",
     "gff2bed1",

lib/galaxy/tools/parameters/basic.py

@@ -296,7 +296,26 @@ def parse_name(input_source):
         return input_source.parse_name()
 
 
-class TextToolParameter(ToolParameter):
+class SimpleTextToolParameter(ToolParameter):
+
+    def __init__(self, tool, input_source):
+        input_source = ensure_input_source(input_source)
+        super().__init__(tool, input_source)
+        self.value = ''
+
+    def to_json(self, value, app, use_security):
+        """Convert a value to a string representation suitable for persisting"""
+        if value is None:
+            rval = ''
+        else:
+            rval = unicodify(value)
+        return rval
+
+    def get_initial_value(self, trans, other_values):
+        return self.value
+
+
+class TextToolParameter(SimpleTextToolParameter):
     """
     Parameter that can take on any text value.
 
@@ -317,22 +336,11 @@ def __init__(self, tool, input_source):
         self.value = input_source.get('value')
         self.area = input_source.get_bool('area', False)
 
-    def to_json(self, value, app, use_security):
-        """Convert a value to a string representation suitable for persisting"""
-        if value is None:
-            rval = ''
-        else:
-            rval = unicodify(value)
-        return rval
-
     def validate(self, value, trans=None):
         search = self.type == "text"
         if not (trans and trans.workflow_building_mode is workflow_building_modes.ENABLED and contains_workflow_parameter(value, search=search)):
             return super().validate(value, trans)
 
-    def get_initial_value(self, trans, other_values):
-        return self.value
-
     def to_dict(self, trans, other_values={}):
         d = super().to_dict(trans)
         d['area'] = self.area
@@ -2328,6 +2336,14 @@ def to_python(self, value, app):
         return json.loads(value)
 
 
+class DirectoryUriToolParameter(SimpleTextToolParameter):
+    """galaxy.files URIs for directories."""
+
+    def __init__(self, tool, input_source, context=None):
+        input_source = ensure_input_source(input_source)
+        SimpleTextToolParameter.__init__(self, tool, input_source)
+
+
 class RulesListToolParameter(BaseJsonToolParameter):
     """
     Parameter that allows for the creation of a list of rules using the Galaxy rules DSL.
@@ -2388,6 +2404,7 @@ def to_text(self, value):
     data_collection=DataCollectionToolParameter,
     library_data=LibraryDatasetToolParameter,
     rules=RulesListToolParameter,
+    directory_uri=DirectoryUriToolParameter,
     drill_down=DrillDownSelectToolParameter
 )