Merge pull request #3 from galaxyproject/release_19.09

Update release 19.09

client/galaxy/scripts/components/Toolshed/Index.vue

@@ -8,8 +8,8 @@
                 @change="setQuery"
                 @keydown.esc="setQuery()"
             />
-            <b-input-group-append>
-                <b-btn :disabled="!queryInput" @click="setQuery()">
+            <b-input-group-append v-b-tooltip.hover title="clear search (esc)">
+                <b-btn @click="setQuery()">
                     <i class="fa fa-times" />
                 </b-btn>
             </b-input-group-append>

client/galaxy/scripts/components/Toolshed/ServerSelection.vue

@@ -4,6 +4,7 @@
         <span v-if="showDropdown" class="dropdown">
             <b-link id="dropdownToolshedUrl" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
                 {{ toolshedUrl }}
+                <span class="fa fa-caret-down" />
             </b-link>
             <div class="dropdown-menu" aria-labelledby="dropdownToolshedUrl">
                 <a
@@ -39,3 +40,8 @@ export default {
     }
 };
 </script>
+<style scoped>
+span.dropdown {
+    display: inline-block;
+}
+</style>

client/galaxy/style/scss/base.scss

@@ -134,7 +134,6 @@ body {
 #right {
     @extend .position-absolute;
     @extend .h-100;
-    @extend .overflow-hidden;
 }
 #left {
     @extend .unified-panel;
@@ -154,7 +153,6 @@ body {
     }
 }
 #center {
-    @extend .overflow-hidden;
     left: $panel-width;
     right: $panel-width;
     .center-container {

config/data_manager_conf.xml.sample

@@ -0,0 +1 @@
+../lib/galaxy/config/sample/data_manager_conf.xml.sample

doc/source/admin/galaxy_options.rst

@@ -235,7 +235,7 @@
     of files, or (for backwards compatibility) a comma-separated list
     of files.
 :Default: ``config/tool_conf.xml``
-:Type: list or str
+:Type: any
 
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -320,7 +320,7 @@
     defaults set relative to this path, such as the default conda
     prefix, default Galaxy packages path, legacy tool shed
     dependencies path, and the dependency cache directory.  Set the
-    string to None to explicitly disable tool dependency handling. If
+    string to null to explicitly disable tool dependency handling. If
     this option is set to none or an invalid path, installing tools
     with dependencies from the Tool Shed or in Conda will fail.
 :Default: ``database/dependencies``
@@ -3217,7 +3217,8 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 :Description:
-    File where Tool Shed based Data Managers are configured.
+    File where Tool Shed based Data Managers are configured. This file
+    will be created automatically upon data manager installation.
 :Default: ``config/shed_data_manager_conf.xml``
 :Type: str
 

doc/source/releases/19.09.rst

@@ -64,7 +64,7 @@ Enhancements
   `Pull Request 7905`_
 * Substantial progress toward refactoring and decomposing Galaxy into properly
   installable packages.
-  (with help from `@dglaetzer <https://github.com/dglaetzer>`__ and
+  (with help from `@dglaetzer <https://github.com/dglaetzer>`__,
   `@nsoranzo <https://github.com/nsoranzo>`__ and `@gmauro <https://github.com/gmauro>`__).
   `Pull Request 921`_, `Pull Request 7992`_, `Pull Request 8114`_,
   `Pull Request 8234`_, `Pull Request 8241`_, `Pull Request 8266`_,
@@ -181,13 +181,11 @@ Enhancements
   `Pull Request 8078`_
 * Allow admins to disable the visualizations tab in the masthead.
   `Pull Request 8450`_
-* Pin SQLAlchemy to ``<1.3`` and update all dependencies
-  `Pull Request 8120`_
+* Update all Python dependencies
+  (thanks to `@nsoranzo <https://github.com/nsoranzo>`__).
+  `Pull Request 8120`_, `Pull Request 8126`_
 * Enable authorization to Google Cloud Platform
   `Pull Request 8123`_
-* Update Python dependencies, unpin SQLAlchemy
-  (thanks to `@nsoranzo <https://github.com/nsoranzo>`__).
-  `Pull Request 8126`_
 * Update requirements and use pyuwsgi
   (thanks to `@mvdbeek <https://github.com/mvdbeek>`__).
   `Pull Request 8489`_
@@ -274,7 +272,7 @@ Enhancements
   `Pull Request 8439`_
 * Fix database migrations for MySQL but mention it is unsupported in the docs
   (thanks to `@nsoranzo <https://github.com/nsoranzo>`__).
-  `Pull Request 8640`_,  `Pull Request 8407`_, `Pull Request 8473`_
+  `Pull Request 8407`_, `Pull Request 8640`_, `Pull Request 8473`_
 * Auto convert python 2 code files using 2to3
   (thanks to `@mvdbeek <https://github.com/mvdbeek>`__).
   `Pull Request 8499`_

doc/source/releases/19.09_announce.rst

@@ -9,30 +9,30 @@ Highlights
 ==========
 
 **Interactive Tools**
-  A new type of Galaxy tool has been implemented that allow interactive experiences like
+  A new type of Galaxy tool has been implemented that allows interactive experiences like
   Galaxy Interactive Environments (via a proxy and Docker containers) but using plugins defined with
   Galaxy's XML tool syntax. Dan Blankenberg's slides on these new features from Galaxy
-  Community Conference can be found `here <https://static.sched.com/hosted_files/gcc2019/c1/S8A-4__Dan_Blankenberg_Tools.pdf>`__.
-  More information can be found in the original pull request including interactive tool
+  Community Conference 2019 can be found `here <https://static.sched.com/hosted_files/gcc2019/c1/S8A-4__Dan_Blankenberg_Tools.pdf>`__.
+  More information can be found in the original pull request, including interactive tool
   versions of all of Galaxy's bundled interactive environments `Pull Request 7494`_.
 
 **Workflow Invocation Reports**
-  Workflows may now define Markdown templates with Galaxy specific embedded syntax extensions
+  Workflows may now define Markdown templates with Galaxy-specific embedded syntax extensions
   that can tie together workflow inputs, outputs, jobs, and metadata into rich reports.
-  More information on the motivation for these reports can be found in `these slides <bit.ly/gxreports19>`__,
+  More information on the motivation for these reports can be found in `these slides <https://bit.ly/gxreports19>`__,
   implementation details and GUI enhancements are described in the original pull request
   (`#8030 <https://github.com/galaxyproject/galaxy/pull/8030>`__), and details on the final
   syntax selected for embedded Galaxy objects can be found in the pull request that was ultimately
   merged (`#8543 <https://github.com/galaxyproject/galaxy/pull/8543>`__).
 
 **New Toolshed Client Interface**
-  The march toward replacing templated backend generated HTML with modern, reactive
+  The march toward replacing templated backend-generated HTML with modern, reactive
   components continued in Galaxy 19.09. Galaxy's tool panel, job properties tables,
   job management controls, and "Pages" infrastructure were all replaced with VueJS
   components (with huge help from `@scrathat <https://github.com/scrathat>`__
   and `@inkuzmin <https://github.com/inkuzmin>`__). Most notably, Galaxy's interface
-  to the Toolshed was also rewritten in VueJS and is now completely API driven.
-  This project drove numerous enhancements to the Toolshed API including vastly improved
+  to the Tool Shed was also rewritten in VueJS and is now completely API driven.
+  This project drove numerous enhancements to the Tool Shed API, including vastly improved
   search functionality. This interface has been completely re-imagined and feels much
   more interactive.
 
@@ -62,34 +62,38 @@ Tool configuration file handling changes
 Shed tool configuration file
 -----------------------------
 
-There have been a few changes to the way that Galaxy loads tool panel configuration files (e.g. `tool_conf.xml` and `shed_tool_conf.xml`) that deployers should be aware of before upgrading.
+There have been a few changes to the way that Galaxy loads tool panel configuration files (e.g. ``tool_conf.xml`` and ``shed_tool_conf.xml``) that deployers should be aware of before upgrading.
 
-A new configuration option named `shed_tool_config_file` has been added, with a default value of `config/shed_tool_conf.xml`, and that path has been removed from the default value of the `tool_config_file` option.  This new option makes it possible for Galaxy to create the shed tool configuration file (e.g.  `shed_tool_conf.xml`) file on demand and moves us one step closer to a Galaxy installation that does not need to be run from the root of its source directory.
+A new configuration option named ``shed_tool_config_file`` has been added, with a default value of ``config/shed_tool_conf.xml``, and that path has been removed from the default value of the ``tool_config_file`` option.  This new option makes it possible for Galaxy to create the shed tool configuration file (e.g. ``shed_tool_conf.xml``) file on demand and moves us one step closer to a Galaxy installation that does not need to be run from the root of its source directory.
 
-If you make no changes to your configuration, you should not encounter any problems. However, if you are using a non-default path to your primary shed tool config file, you should modify these options to remove the path of your primary `shed_tool_conf.xml` from `tool_config_file` and set it in `shed_tool_config_file`. Although Galaxy will continue to operate properly if you do not make this change, failure to do so will result in a second shed tool panel config file being created and loaded at startup time.
+If you make no changes to your configuration, you should not encounter any problems. However, if you are using a non-default path to your primary shed tool config file, you should modify these options to remove the path of your primary ``shed_tool_conf.xml`` from ``tool_config_file`` and set it in ``shed_tool_config_file``. Although Galaxy will continue to operate properly if you do not make this change, failure to do so will result in a second shed tool panel config file being created and loaded at startup time.
 
-Although an unusual configuration, if you have multiple shed tool config files, you can still load them in `tool_config_file`, just be sure that one of them is set in `shed_tool_config_file`.
+Although an unusual configuration, if you have multiple shed tool config files, you can still load them in ``tool_config_file``, just be sure that one of them is set in ``shed_tool_config_file``.
 
-Note that Galaxy will fail to start if any of the files specified in `tool_conf_file` cannot be read at startup, whereas it will only fail to start if the file specified in `shed_tool_config_file` cannot be created (if it does not already exist).
+Note that Galaxy will fail to start if any of the files specified in ``tool_conf_file`` cannot be read at startup, whereas it will only fail to start if the file specified in ``shed_tool_config_file`` cannot be created (if it does not already exist).
 
 Migrated tools configuration file
 ---------------------------------
 
-Galaxy contains a (now unused) system for automatically installing tools from the Tool Shed that had been migrated out of the Galaxy source code after the creation of the Tool Shed. Older deployments that used this system may have tools in the migrated tools config file, `config/migrated_tools_conf.xml.sample` (by default). As of this release, an empty `migrated_tools_conf.xml` file is not created by default for new installations, but will still be read if it is found at the default path. The path can be changed with the `migrated_tools_config` option, and you can force Galaxy to fail startup if the file is missing by adding it to the `tool_config_file` option.
+Galaxy contains a (now unused) system for automatically installing tools from the Tool Shed that had been migrated out of the Galaxy source code after the creation of the Tool Shed. Older deployments that used this system may have tools in the migrated tools config file, ``config/migrated_tools_conf.xml.sample`` (by default). As of this release, an empty ``migrated_tools_conf.xml`` file is not created by default for new installations, but will still be read if it is found at the default path. The path can be changed with the ``migrated_tools_config`` option, and you can force Galaxy to fail startup if the file is missing by adding it to the ``tool_config_file`` option.
 
 Configuration file path defaults
 --------------------------------
 
 **Deprecated paths will no longer be searched**
-The default location for Galaxy's configuration files has been in the `config/` directory for many years, prior to which the default was the root of the Galaxy directory. As of this release, Galaxy will no longer look in the current directory for configuration files, so you will need to specify the path to `galaxy.yml` e.g. with `uwsgi --yaml /path/to/galaxy.yml`, `GALAXY_CONFIG_FILE=/path/to/galaxy.yml run.sh`, or similar means, if it is not in the default location.
+The default location for Galaxy's configuration files has been in the ``config/`` directory for many years, prior to which the default was the root of the Galaxy directory.
+As of this release, Galaxy will no longer look in the current directory for configuration files, so you will need to specify the path to ``galaxy.yml`` e.g. with
+``uwsgi --yaml /path/to/galaxy.yml``, ``GALAXY_CONFIG_FILE=/path/to/galaxy.yml run.sh``, or similar means, if it is not in the default location.
 
 **Default paths to additional configuration files**
-As of this release, all other configuration files (e.g. `datatypes_conf.xml`, `job_conf.xml`) are searched for in the same directory as `galaxy.yml`. Prior to this release, the default was in the `config/` directory of the current working directory at startup (which has always been, and for the time being continues to be, the root of the Galaxy source).  This means that you do not need to individually configure the paths to these files in `galaxy.yml` if you are storing them in non-default locations.
+As of this release, all other configuration files (e.g. ``datatypes_conf.xml``, ``job_conf.xml``) are searched for in the same directory as ``galaxy.yml``.
+Prior to this release, the default was in the ``config/`` directory of the current working directory at startup (which has always been, and for the time being continues to be, the root of the Galaxy source).
+This means that you do not need to individually configure the paths to these files in ``galaxy.yml`` if you are storing them in non-default locations.
 
 MySQL Support Drop
 ==================
 
-Going forward the MySQL database server is no longer supported and all deployers are urged to use PostgreSQL.
+The MySQL database server is no longer officially supported and all deployers are urged to use PostgreSQL.
 
 Deprecation Notice
 ==================
@@ -105,23 +109,23 @@ The following Galaxy API endpoints (related to the Toolshed) are considered depr
 Python
 ------
 
-TL;DR Galaxy can be deployed on both Python 2 and 3 with Python3 expected to be the increasingly encouraged way of deployment.
+TL;DR Galaxy can be deployed on both Python 2 and 3, with Python 3 expected to be the increasingly encouraged way of deployment.
 
-Python2 will `not be maintained <https://www.python.org/doc/sunset-python-2/>`__ past 2020 which means Galaxy 19.09 is
+Python 2 will `not be maintained <https://www.python.org/doc/sunset-python-2/>`__ past 2020 which means Galaxy 19.09 is
 the last release that officially supports Python 2.
 
-Over the past years Galaxy team has made a substantial effort
-spearheaded by `@nsoranzo <https://github.com/nsoranzo>`__ and `@mvdbeek <https://github.com/mvdbeek>`__
-to make Galaxy compatible with Python3. Thanks to this endeavor the `Galaxy Test server <https://test.galaxyproject.org>`__
-has been running on Python3 since April of this year and **Galaxy itself now fully supports Python3**.
+Over the past years the Galaxy team has made a substantial effort,
+spearheaded by `@nsoranzo <https://github.com/nsoranzo>`__ and `@mvdbeek <https://github.com/mvdbeek>`__,
+to make Galaxy compatible with Python 3. Thanks to this endeavor the `Galaxy Test server <https://test.galaxyproject.org>`__
+has been running on Python 3 since April of this year and **Galaxy itself now fully supports Python 3**.
 
-**Galaxy Tools**, that properly declare their Python2 dependency (using <requirements>), will continue to work as expected with Python2
-being provided by Conda. However tools that naively rely on implicit presence of Python2 on the PATH can fail on Python3 deployments.
+**Galaxy Tools**, that properly declare their Python dependency (using e.g. ``<requirement type="package" version="2.7">python</requirement>``), will continue to work as expected with Python 2
+being provided by Conda. However tools that naively rely on the implicit presence of Python 2 on the PATH may fail on Python3 deployments.
 Another corner case are tools that use Python2-specific Cheetah templating syntax in their XML. We have implemented various mitigation
-approaches on the Galaxy side but some old tools using this syntax will not function correctly.
+approaches on the Galaxy side, but some old tools using this syntax will not function correctly.
 
 *In our testing these problems should affect only a very limited number of tools and the vast majority should
-work on Python3 without a change.*
+work on Python 3 without a change.*
 
 
 Release Notes

lib/galaxy/actions/library.py

@@ -75,6 +75,11 @@ def _upload_dataset(self, trans, library_id, folder_id, replace_dataset=None, **
         cntrller = 'api'
         tool_id = 'upload1'
         message = None
+        file_type = kwd.get('file_type')
+        try:
+            upload_common.validate_datatype_extension(datatypes_registry=trans.app.datatypes_registry, ext=file_type)
+        except RequestParameterInvalidException as e:
+            return (400, util.unicodify(e))
         tool = trans.app.toolbox.get_tool(tool_id)
         state = tool.new_state(trans)
         populate_state(trans, tool.inputs, kwd, state.inputs)

lib/galaxy/config/__init__.py

@@ -844,7 +844,7 @@ def parse_config_file_options(self, kwargs):
             tool_sheds_config_file=[self._in_config_dir('tool_sheds_conf.xml')],
             user_preferences_extra_conf_path=[self._in_config_dir('user_preferences_extra_conf.yml')],
             workflow_resource_params_file=[self._in_config_dir('workflow_resource_params_conf.xml')],
-            workflow_schedulers_config_file=[self._in_config_dir('config/workflow_schedulers_conf.xml')],
+            workflow_schedulers_config_file=[self._in_config_dir('workflow_schedulers_conf.xml')],
         )
         listify_defaults = {
             'tool_data_table_config_path': [

lib/galaxy/config/sample/data_manager_conf.xml.sample

@@ -0,0 +1,3 @@
+<?xml version="1.0"?>
+<data_managers>
+</data_managers>

lib/galaxy/config/sample/galaxy.yml.sample

@@ -242,7 +242,7 @@ galaxy:
   # Various dependency resolver configuration parameters will have
   # defaults set relative to this path, such as the default conda
   # prefix, default Galaxy packages path, legacy tool shed dependencies
-  # path, and the dependency cache directory.  Set the string to None to
+  # path, and the dependency cache directory.  Set the string to null to
   # explicitly disable tool dependency handling. If this option is set
   # to none or an invalid path, installing tools with dependencies from
   # the Tool Shed or in Conda will fail.
@@ -1543,7 +1543,8 @@ galaxy:
   # does not exist).
   #data_manager_config_file: config/data_manager_conf.xml
 
-  # File where Tool Shed based Data Managers are configured.
+  # File where Tool Shed based Data Managers are configured. This file
+  # will be created automatically upon data manager installation.
   #shed_data_manager_config_file: config/shed_data_manager_conf.xml
 
   # Directory to store Data Manager based tool-data; defaults to

lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt

@@ -3,7 +3,6 @@
 adal==1.2.2
 amqp==2.5.1
 appdirs==1.4.3
-asn1crypto==0.24.0
 attrs==19.1.0
 avro==1.8.1 ; python_version < '3'
 azure-common==1.1.14
@@ -44,7 +43,7 @@ cloudbridge==2.0.0
 cmd2==0.8.9
 configparser==3.8.1 ; python_version < '3.2'
 contextlib2==0.5.5 ; python_version < '3.5'
-cryptography==2.7
+cryptography==2.8
 cwltool==1.0.20180721142728
 debtcollector==1.21.0
 decorator==4.4.0

lib/galaxy/metadata/set_metadata.py

@@ -85,7 +85,9 @@ def set_metadata():
 
 def set_metadata_portable():
     import galaxy.model
-    galaxy.model.metadata.MetadataTempFile.tmp_dir = tool_job_working_directory = os.path.abspath(os.getcwd())
+    tool_job_working_directory = os.path.abspath(os.getcwd())
+    metadata_tmp_files_dir = os.path.join(tool_job_working_directory, "metadata")
+    galaxy.model.metadata.MetadataTempFile.tmp_dir = metadata_tmp_files_dir
 
     metadata_params_path = os.path.join("metadata", "params.json")
     try: