Integrate RuleBuilder with remote_files plugins.

- Extend FileDialog to allow selection of directories. - Add new rule source that is a remote directory, pre-loaded into the rule builder with URL column assigned from the metadata.
galaxyproject · Jun 29, 2020 · 5225a00 · 5225a00
1 parent 9a29b4e
commit 5225a00
Show file tree

Hide file tree

Showing 7 changed files with 110 additions and 23 deletions.
diff --git a/client/galaxy/scripts/components/FilesDialog/FilesDialog.vue b/client/galaxy/scripts/components/FilesDialog/FilesDialog.vue
@@ -16,7 +16,9 @@
                 :filter="filter"
                 :showDetails="showDetails"
                 :showTime="showTime"
+                :showNavigate="mode == 'directory'"
                 @clicked="clicked"
+                @open="open"
                 @load="load"
             />
         </template>
@@ -53,6 +55,11 @@ export default {
             type: Boolean,
             default: false,
         },
+        mode: {
+            type: String,
+            default: "file",
+            validator: (prop) => ["file", "directory"].includes(prop),
+        },
     },
     data() {
         return {
@@ -73,20 +80,25 @@ export default {
         this.model = new Model({ multiple: this.multiple });
         this.load();
     },
+    computed: {
+        fileMode() {
+            return this.mode == "file";
+        },
+    },
     methods: {
         /** Add highlighting for record variations, i.e. datasets vs. libraries/collections **/
         formatRows() {
             for (const item of this.items) {
                 let _rowVariant = "active";
-                if (item.isLeaf) {
+                if (item.isLeaf || !this.fileMode) {
                     _rowVariant = this.model.exists(item.id) ? "success" : "default";
                 }
                 Vue.set(item, "_rowVariant", _rowVariant);
             }
         },
         /** Collects selected datasets in value array **/
         clicked: function (record) {
-            if (record.isLeaf) {
+            if (record.isLeaf || !this.fileMode) {
                 this.model.add(record);
                 this.hasValue = this.model.count() > 0;
                 if (this.multiple) {
@@ -95,9 +107,12 @@ export default {
                     this.finalize();
                 }
             } else {
-                this.load(record.url);
+                this.open(record);
             }
         },
+        open: function (record) {
+            this.load(record.url);
+        },
         /** Called when selection is complete, values are formatted and parsed to external callback **/
         finalize: function () {
             const results = this.model.finalize();
@@ -136,18 +151,35 @@ export default {
                 this.services
                     .list(url)
                     .then((items) => {
-                        this.items = items.map((item) => {
-                            const itemClass = item.class;
-                            return {
-                                id: item.uri,
-                                label: item.name,
-                                time: item.ctime,
-                                isLeaf: itemClass == "File",
-                                size: item.size,
-                                url: item.uri,
-                                labelTitle: item.uri,
-                            };
-                        });
+                        if (this.fileMode) {
+                            items = items.map((item) => {
+                                const itemClass = item.class;
+                                return {
+                                    id: item.uri,
+                                    label: item.name,
+                                    time: item.ctime,
+                                    isLeaf: itemClass == "File",
+                                    size: item.size,
+                                    url: item.uri,
+                                    labelTitle: item.uri,
+                                };
+                            });
+                        } else {
+                            items = items
+                                .filter((item) => item.class == "Directory")
+                                .map((item) => {
+                                    return {
+                                        id: item.uri,
+                                        label: item.name,
+                                        time: item.ctime,
+                                        isLeaf: false,
+                                        size: item.size,
+                                        url: item.uri,
+                                        labelTitle: item.uri,
+                                    };
+                                });
+                        }
+                        this.items = items;
                         this.formatRows();
                         this.optionsShow = true;
                         this.showTime = true;

diff --git a/client/galaxy/scripts/components/FilesDialog/model.js b/client/galaxy/scripts/components/FilesDialog/model.js
@@ -38,8 +38,7 @@ export class Model {
     finalize() {
         let results = [];
         Object.values(this.values).forEach((v) => {
-            let value = v;
-            results.push(value);
+            results.push(v);
         });
         if (results.length > 0 && !this.multiple) {
             results = results[0];

diff --git a/client/galaxy/scripts/components/RuleCollectionBuilder.vue b/client/galaxy/scripts/components/RuleCollectionBuilder.vue
@@ -629,6 +629,8 @@ export default {
         } else {
             if (this.elementsType == "ftp") {
                 mapping = [{ type: "ftp_path", columns: [0] }];
+            } else if (this.elementsType == "remote_files") {
+                mapping = [{ type: "url", columns: [0] }];
             } else if (this.elementsType == "datasets") {
                 mapping = [{ type: "list_identifiers", columns: [1] }];
             } else {
@@ -674,6 +676,11 @@ export default {
                     type: "add_column_metadata",
                     value: "path",
                 });
+            } else if (this.elementsType == "remote_files") {
+                rules.push({
+                    type: "add_column_metadata",
+                    value: "uri",
+                });
             }
         }
         return {
@@ -961,6 +968,9 @@ export default {
                 metadataOptions["tags"] = _l("Tags");
             } else if (this.elementsType == "ftp") {
                 metadataOptions["path"] = _l("Path");
+            } else if (this.elementsType == "remote_files") {
+                // IS THIS NEEDED?
+                metadataOptions["url"] = _l("URL");
             } else if (this.elementsType == "library_datasets") {
                 metadataOptions["name"] = _l("Name");
             } else if (this.elementsType == "datasets") {
@@ -1007,7 +1017,8 @@ export default {
                 valid = false;
             }
 
-            const requiresSourceColumn = this.elementsType == "ftp" || this.elementsType == "raw";
+            const requiresSourceColumn =
+                this.elementsType == "ftp" || this.elementsType == "raw" || this.elementsType == "remote_files";
             if (requiresSourceColumn && !mappingAsDict.ftp_path && !mappingAsDict.url) {
                 valid = false;
             }
@@ -1029,7 +1040,8 @@ export default {
             if (
                 this.elementsType == "datasets" ||
                 this.elementsType == "library_datasets" ||
-                this.elementsType == "ftp"
+                this.elementsType == "ftp" ||
+                this.elementsType == "remote_files"
             ) {
                 sources = this.initialElements.slice();
                 data = sources.map((el) => []);

diff --git a/client/galaxy/scripts/components/SelectionDialog/DataDialogTable.vue b/client/galaxy/scripts/components/SelectionDialog/DataDialogTable.vue
@@ -21,6 +21,9 @@
             <template v-slot:cell(time)="data">
                 {{ data.value ? data.value : "-" }}
             </template>
+            <template v-slot:cell(navigate)="data">
+                <i v-if="!data.item.isLeaf" class="fa fa-caret-square-o-right" @click.stop="open(data.item)" />
+            </template>
         </b-table>
         <div v-if="nItems === 0">
             <div v-if="filter">
@@ -42,6 +45,7 @@ Vue.use(BootstrapVue);
 const LABEL_FIELD = { key: "label", sortable: true };
 const DETAILS_FIELD = { key: "details", sortable: true };
 const TIME_FIELD = { key: "time", sortable: true };
+const NAVIGATE_FIELD = { key: "navigate", label: "", sortable: false };
 
 export default {
     props: {
@@ -69,6 +73,10 @@ export default {
             type: Boolean,
             default: true,
         },
+        showNavigate: {
+            type: Boolean,
+            default: false,
+        },
     },
     data() {
         return {
@@ -94,6 +102,9 @@ export default {
             if (this.showTime) {
                 fields.push(TIME_FIELD);
             }
+            if (this.showNavigate) {
+                fields.push(NAVIGATE_FIELD);
+            }
             return fields;
         },
     },
@@ -107,6 +118,9 @@ export default {
         clicked: function (record) {
             this.$emit("clicked", record);
         },
+        open: function (record) {
+            this.$emit("open", record);
+        },
     },
 };
 </script>
diff --git a/client/galaxy/scripts/components/Upload/RulesInput.vue b/client/galaxy/scripts/components/Upload/RulesInput.vue
@@ -16,8 +16,9 @@
                     <select2 container-class="upload-footer-selection" v-model="selectionType">
                         <option value="paste">{{ l("Pasted Table") }}</option>
                         <option value="dataset">{{ l("History Dataset") }}</option>
-                        <option v-if="ftpUploadSite" value="ftp">{{ l("FTP Directory") }} </option></select2
-                    >
+                        <option v-if="ftpUploadSite" value="ftp">{{ l("FTP Directory") }} </option>
+                        <option value="remote_files">{{ l("Remote Files Directory") }}</option>
+                    </select2>
                 </div>
             </div>
             <div id="upload-rule-dataset-option" class="upload-rule-option" v-if="selectionType == 'dataset'">
@@ -80,6 +81,7 @@ import BootstrapVue from "bootstrap-vue";
 import UploadUtils from "mvc/upload/upload-utils";
 import axios from "axios";
 import { getAppRoot } from "onload/loadConfig";
+import { filesDialog } from "utils/data";
 
 Vue.use(BootstrapVue);
 
@@ -90,6 +92,7 @@ export default {
             l: _l,
             datasets: [],
             ftpFiles: [],
+            uris: [],
             datasetsSet: false,
             topInfo: _l("Tabular source data to extract collection files and metadata from"),
             enableReset: false,
@@ -126,6 +129,8 @@ export default {
                     this.sourceContent = ftp_files.map((file) => file["path"]).join("\n");
                     this.ftpFiles = ftp_files;
                 });
+            } else if (selectionType == "remote_files") {
+                filesDialog(this._handleRemoteFilesUri, { mode: "directory" });
             }
         },
         selectedDatasetId: function (selectedDatasetId) {
@@ -152,6 +157,15 @@ export default {
             this.sourceContent = "";
         },
 
+        _handleRemoteFilesUri: function (record) {
+            // fetch files at URI
+            UploadUtils.getRemoteFilesAt(record.url).then((files) => {
+                files = files.filter((file) => file["class"] == "File");
+                this.sourceContent = files.map((file) => file["uri"]).join("\n");
+                this.uris = files;
+            });
+        },
+
         _eventBuild: function () {
             this._buildSelection(this.sourceContent);
         },
@@ -167,6 +181,9 @@ export default {
                 selection.selectionType = "ftp";
                 selection.elements = this.ftpFiles;
                 selection.ftpUploadSite = this.ftpUploadSite;
+            } else if (selectionType == "remote_files") {
+                selection.selectionType = "remote_files";
+                selection.elements = this.uris;
             }
             selection.dataType = this.dataType;
             Galaxy.currHistoryPanel.buildCollection("rules", selection, true);

diff --git a/client/galaxy/scripts/mvc/rules/rule-definitions.js b/client/galaxy/scripts/mvc/rules/rule-definitions.js
@@ -183,14 +183,14 @@ const RULES = {
                     newRow.push(tags.join(","));
                     return newRow;
                 };
-            } else if (ruleValue == "hid" || ruleValue == "name" || ruleValue == "path") {
+            } else if (ruleValue == "hid" || ruleValue == "name" || ruleValue == "path" || ruleValue == "uri") {
                 newRow = (row, index) => {
                     const newRow = row.slice();
                     newRow.push(sources[index][ruleValue]);
                     return newRow;
                 };
             } else {
-                return { error: `Unknown metadata type [${ruleValue}}]` };
+                return { error: `Unknown metadata type [${ruleValue}]` };
             }
             data = data.map(newRow);
             columns.push(NEW_COLUMN);

diff --git a/client/galaxy/scripts/mvc/upload/upload-utils.js b/client/galaxy/scripts/mvc/upload/upload-utils.js
@@ -1,6 +1,7 @@
 import $ from "jquery";
 import { getAppRoot } from "onload/loadConfig";
 import axios from "axios";
+import { rethrowSimple } from "utils/simple-error";
 
 const AUTO_EXTENSION = {
     id: "auto",
@@ -72,6 +73,17 @@ function getUploadGenomes(callback, defaultGenome) {
         });
 }
 
+async function getRemoteFilesAt(target) {
+    const url = `${getAppRoot()}api/remote_files?target=${target}`;
+    try {
+        const response = await axios.get(url);
+        const files = response.data;
+        return files;
+    } catch (e) {
+        rethrowSimple(e);
+    }
+}
+
 function getRemoteFiles(success, error) {
     return $.ajax({
         url: `${getAppRoot()}api/remote_files`,
@@ -86,6 +98,7 @@ export default {
     DEFAULT_GENOME,
     DEFAULT_EXTENSION,
     getRemoteFiles,
+    getRemoteFilesAt,
     getUploadDatatypes,
     getUploadGenomes,
 };