Add infrastructure for writing to pluggable galaxy file sources.

client/src/mvc/form/form-parameters.js

@@ -10,6 +10,7 @@ import SelectContent from "mvc/ui/ui-select-content";
 import SelectLibrary from "mvc/ui/ui-select-library";
 import SelectFtp from "mvc/ui/ui-select-ftp";
 import RulesEdit from "mvc/ui/ui-rules-edit";
+import FileSource from "mvc/ui/ui-file-source";
 import ColorPicker from "mvc/ui/ui-color-picker";
 import DataPicker from "mvc/ui/ui-data-picker";
 
@@ -37,6 +38,7 @@ export default Backbone.Model.extend({
         ftpfile: "_fieldFtp",
         upload: "_fieldUpload",
         rules: "_fieldRulesEdit",
+        directory_uri: "_fieldDirectoryUri",
         data_dialog: "_fieldDialog",
     },
 
@@ -238,6 +240,14 @@ export default Backbone.Model.extend({
         });
     },
 
+    _fieldDirectoryUri: function (input_def) {
+        return new FileSource.View({
+            id: `field-${input_def.id}`,
+            onchange: input_def.onchange,
+            target: input_def.target,
+        });
+    },
+
     _fieldRulesEdit: function (input_def) {
         return new RulesEdit.View({
             id: `field-${input_def.id}`,

client/src/mvc/ui/ui-file-source.js

@@ -0,0 +1,100 @@
+import Backbone from "backbone";
+import { filesDialog } from "utils/data";
+import _l from "utils/localization";
+import Ui from "mvc/ui/ui-misc";
+
+var View = Backbone.View.extend({
+    initialize: function (options) {
+        this.model = new Backbone.Model();
+        this.target = options.target;
+
+        // create insert edit button
+        this.browse_button = new Ui.Button({
+            title: _l("Select"),
+            icon: "fa fa-edit",
+            tooltip: _l("Select URI"),
+            onclick: () => {
+                filesDialog(
+                    (uri) => {
+                        this._handleRemoteFilesUri(uri);
+                    },
+                    { mode: "directory" }
+                );
+            },
+        });
+
+        // add change event. fires on trigger
+        this.on("change", () => {
+            if (options.onchange) {
+                options.onchange(this.value());
+            }
+        });
+
+        // create elements
+        this.setElement(this._template(options));
+        this.$text = this.$(".ui-uri-preview");
+        this.$(".ui-file-select-button").append(this.browse_button.$el);
+        this.listenTo(this.model, "change", this.render, this);
+        this.render();
+    },
+
+    _handleRemoteFilesUri: function (uri) {
+        this._setValue(uri);
+    },
+
+    render: function () {
+        const value = this._value;
+        if (value) {
+            if (value.url !== this.$text.text()) {
+                this.$text.text(value.url);
+            }
+        } else {
+            this.$text.text("select...");
+        }
+    },
+
+    /** Main Template */
+    _template: function (options) {
+        return `
+            <div class="ui-rules-edit clearfix">
+                <span class="ui-uri-preview" />
+                <span class="ui-file-select-button float-left" />
+            </div>
+        `;
+    },
+
+    /** Return/Set current value */
+    value: function (new_value) {
+        if (new_value !== undefined) {
+            this._setValue(new_value);
+        } else {
+            return this._getValue();
+        }
+    },
+
+    /** Update input element options */
+    update: function (input_def) {
+        this.target = input_def.target;
+    },
+
+    /** Returns current value */
+    _getValue: function () {
+        return this._value.url;
+    },
+
+    /** Sets current value */
+    _setValue: function (new_value) {
+        if (new_value) {
+            if (typeof new_value == "string") {
+                new_value = JSON.parse(new_value);
+            }
+            this._value = new_value;
+            this.model.trigger("error", null);
+            this.model.trigger("change");
+        }
+    },
+});
+
+export default {
+    View: View,
+};

lib/galaxy/files/sources/__init__.py

@@ -6,6 +6,9 @@
 
 from galaxy.util.template import fill_template
 
+DEFAULT_SCHEME = "gxfiles"
+DEFAULT_WRITABLE = False
+
 
 @six.add_metaclass(abc.ABCMeta)
 class FilesSource(object):
@@ -20,6 +23,10 @@ def get_uri_root(self):
     def get_scheme(self):
         """Return a prefix for the root (e.g. the gxfiles in gxfiles://prefix/path)."""
 
+    @abc.abstractmethod
+    def get_writable(self):
+        """Return a boolean indicating if this target is writable."""
+
     # TODO: off-by-default
     @abc.abstractmethod
     def list(self, source_path="/", recursive=False, user_context=None):
@@ -29,6 +36,10 @@ def list(self, source_path="/", recursive=False, user_context=None):
     def realize_to(self, source_path, native_path, user_context=None):
         """Realize source path (relative to uri root) to local file system path."""
 
+    def write_from(self, target_path, native_path, user_context=None):
+        """Write file at native path to target_path (relative to uri root).
+        """
+
     @abc.abstractmethod
     def to_dict(self, for_serialization=False, user_context=None):
         """Return a dictified representation of this FileSource instance.
@@ -46,6 +57,9 @@ def get_prefix(self):
     def get_scheme(self):
         return "gxfiles"
 
+    def get_writable(self):
+        return self.writable
+
     def get_uri_root(self):
         prefix = self.get_prefix()
         scheme = self.get_scheme()
@@ -63,7 +77,8 @@ def _parse_common_config_opts(self, kwd):
         self.id = kwd.pop("id")
         self.label = kwd.pop("label", None) or self.id
         self.doc = kwd.pop("doc", None)
-        self.scheme = kwd.pop("scheme", "gxfiles")
+        self.scheme = kwd.pop("scheme", DEFAULT_SCHEME)
+        self.writable = kwd.pop("writable", DEFAULT_WRITABLE)
         # If coming from to_dict, strip API helper values
         kwd.pop("uri_root", None)
         kwd.pop("type", None)
@@ -96,6 +111,15 @@ def _serialization_props(self):
         Used in to_dict method if for_serialization is True.
         """
 
+    def write_from(self, target_path, native_path, user_context=None):
+        if not self.get_writable():
+            raise Exception("Cannot write to a non-writable file source.")
+        self._write_from(target_path, native_path, user_context=user_context)
+
+    @abc.abstractmethod
+    def _write_from(self, target_path, native_path, user_context=None):
+        pass
+
     def _evaluate_prop(self, prop_val, user_context):
         rval = prop_val
         if "$" in prop_val:

lib/galaxy/files/sources/galaxy.py

@@ -12,6 +12,7 @@ def __init__(self, label="FTP Directory", doc="Galaxy User's FTP Directory", roo
             root=root,
             label=label,
             doc=doc,
+            writable=True,
         )
         posix_kwds.update(kwd)
         if "delete_on_realize" not in posix_kwds:

lib/galaxy/files/sources/posix.py

@@ -62,6 +62,16 @@ def realize_to(self, source_path, native_path, user_context=None):
         else:
             shutil.move(source_native_path, native_path)
 
+    def _write_from(self, target_path, native_path, user_context=None):
+        effective_root = self._effective_root(user_context)
+        target_native_path = self._to_native_path(target_path, user_context=user_context)
+        # TODO: enforce_symlink_security
+        target_native_path = os.path.normpath(target_native_path)
+        assert target_native_path.startswith(os.path.normpath(effective_root))
+
+        # TODO: ensure directory exists, etc...
+        shutil.copyfile(native_path, target_native_path)
+
     def _to_native_path(self, source_path, user_context=None):
         source_path = os.path.normpath(source_path)
         if source_path.startswith("/"):

lib/galaxy/tool_util/xsd/galaxy.xsd

@@ -2398,6 +2398,14 @@ sanitizer to prevent Galaxy from escaping the result.
 </param>
 ```
 
+#### ``directory_uri``
+
+This is used to tie into galaxy.files URI infrastructure. This should only be used by
+core Galaxy tools until the interface around files has stabilized.
+
+Currently ``directory_uri`` parameters provide user's the option of selecting a writable
+directory destination for unstructured outputs of tools (e.g. history exports).
+
 $attribute_list:value,rgb:5
 
 This covers examples of the most common parameter types, the remaining parameter
@@ -2693,6 +2701,7 @@ allow access to Python code to generate options for a select list. See
       <xs:enumeration value="drill_down"/>
       <xs:enumeration value="group_tag"/>
       <xs:enumeration value="data_collection"/>
+      <xs:enumeration value="directory_uri" />
     </xs:restriction>
   </xs:simpleType>
 

lib/galaxy/tools/__init__.py

@@ -128,6 +128,7 @@
     "upload1",
     "send_to_cloud",
     "__DATA_FETCH__",
+    "directory_uri",
     # Legacy tools bundled with Galaxy.
     "laj_1",
     "gff2bed1",

lib/galaxy/tools/parameters/basic.py

@@ -258,7 +258,26 @@ def parse_name(input_source):
         return input_source.parse_name()
 
 
-class TextToolParameter(ToolParameter):
+class SimpleTextToolParameter(ToolParameter):
+
+    def __init__(self, tool, input_source):
+        input_source = ensure_input_source(input_source)
+        super().__init__(tool, input_source)
+        self.value = ''
+
+    def to_json(self, value, app, use_security):
+        """Convert a value to a string representation suitable for persisting"""
+        if value is None:
+            rval = ''
+        else:
+            rval = unicodify(value)
+        return rval
+
+    def get_initial_value(self, trans, other_values):
+        return self.value
+
+
+class TextToolParameter(SimpleTextToolParameter):
     """
     Parameter that can take on any text value.
 
@@ -279,22 +298,11 @@ def __init__(self, tool, input_source):
         self.value = input_source.get('value')
         self.area = input_source.get_bool('area', False)
 
-    def to_json(self, value, app, use_security):
-        """Convert a value to a string representation suitable for persisting"""
-        if value is None:
-            rval = ''
-        else:
-            rval = unicodify(value)
-        return rval
-
     def validate(self, value, trans=None):
         search = self.type == "text"
         if not (trans and trans.workflow_building_mode is workflow_building_modes.ENABLED and contains_workflow_parameter(value, search=search)):
             return super().validate(value, trans)
 
-    def get_initial_value(self, trans, other_values):
-        return self.value
-
     def to_dict(self, trans, other_values={}):
         d = super().to_dict(trans)
         d['area'] = self.area
@@ -2293,6 +2301,14 @@ def to_python(self, value, app):
         return json.loads(value)
 
 
+class DirectoryUriToolParameter(SimpleTextToolParameter):
+    """galaxy.files URIs for directories."""
+
+    def __init__(self, tool, input_source, context=None):
+        input_source = ensure_input_source(input_source)
+        SimpleTextToolParameter.__init__(self, tool, input_source)
+
+
 class RulesListToolParameter(BaseJsonToolParameter):
     """
     Parameter that allows for the creation of a list of rules using the Galaxy rules DSL.
@@ -2353,6 +2369,7 @@ def to_text(self, value):
     data_collection=DataCollectionToolParameter,
     library_data=LibraryDatasetToolParameter,
     rules=RulesListToolParameter,
+    directory_uri=DirectoryUriToolParameter,
     drill_down=DrillDownSelectToolParameter
 )
 

test/functional/tools/directory_uri.xml

@@ -0,0 +1,17 @@
+<tool id="directory_uri" name="directory_uri" version="1.0.0">
+    <command><![CDATA[
+        python '$__tool_directory__/directory_uri_copy_to.py'
+            --file_sources '$file_sources'
+            --directory_uri '$d_uri'
+    ]]></command>
+    <inputs>
+        <param type="directory_uri" name="d_uri" label="Directory URI" />
+    </inputs>
+    <outputs>
+    </outputs>
+    <configfiles>
+        <file_sources name="file_sources" />
+    </configfiles>
+    <tests>
+    </tests>
+</tool>

test/functional/tools/directory_uri_copy_to.py

@@ -0,0 +1,49 @@
+import argparse
+import json
+import os
+import sys
+import tempfile
+
+from galaxy.files import ConfiguredFileSources
+
+
+def get_file_sources(file_sources_path):
+    assert os.path.exists(file_sources_path), "file sources path [%s] does not exist" % file_sources_path
+    with open(file_sources_path, "r") as f:
+        with open("/Users/john/workspace/galaxy/cowdog", "w") as g:
+            g.write(f.read())
+    with open(file_sources_path, "r") as f:
+        file_sources_as_dict = json.load(f)
+    file_sources = ConfiguredFileSources.from_dict(file_sources_as_dict)
+    return file_sources
+
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv[1:]
+
+    args = _parser().parse_args(argv)
+    file_sources = get_file_sources(args.file_sources)
+    directory_uri = args.directory_uri
+    if directory_uri.endswith("/"):
+        target_uri = directory_uri + "helloworld"
+    else:
+        target_uri = directory_uri + "/helloworld"
+    file_source_path = file_sources.get_file_source_path(target_uri)
+    file_source = file_source_path.file_source
+    fd, temp_name = tempfile.mkstemp()
+    with open(fd, 'w') as f:
+        f.write('hello world!\n')
+    file_source.write_from(file_source_path.path, temp_name)
+
+
+def _parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--directory_uri', type=str,
+                        help='directory target URI')
+    parser.add_argument('--file_sources', type=str, help='file sources json')
+    return parser
+
+
+if __name__ == "__main__":
+    main()