Merge remote-tracking branch 'remotes/main/dev' into UserBasedObjectS…

…tore

# Conflicts:
#	doc/source/lib/galaxy.objectstore.rst
#	doc/source/slideshow/architecture/images/objectstore.plantuml.svg
#	lib/galaxy/jobs/__init__.py

.ci/flake8_blacklist.txt

@@ -5,3 +5,4 @@ database/
 doc/source/conf.py
 eggs/
 lib/galaxy/util/jstree.py
+lib/galaxy/web/proxy/js/node_modules/

.ci/flake8_docstrings_include_list.txt

@@ -1,2 +1,3 @@
 lib/galaxy/jobs/metrics
 lib/galaxy/exceptions
+lib/galaxy/containers

.ci/flake8_wrapper_docstrings.sh

@@ -7,7 +7,7 @@ set -e
 # D3XX - Quoting issues.
 # D401 - First line should be in imperative mood 
 # D403 - First word of the first line should be properly capitalized
-args="--ignore=D --select=D100,D201,D202,D206,D207,D208,D209,D211,D3,D401,D403"
+args="--ignore=D --select=D100,D201,D202,D206,D207,D208,D209,D211,D3,D403"
 
 # If the first argument is --include, lint the modules expected to pass. If
 # the first argument is --exclude, lint all modules the full Galaxy linter lints

.ci/jenkins/selenium/run_galaxy.bash

@@ -2,9 +2,12 @@
 
 set -e
 
-sleep 30  # TODO: wait on something instead of just sleeping...
-
-echo `df`
+echo "Waiting for postgres to become available"
+while ! nc -z postgres 5432;
+do
+    sleep 1
+    printf "."
+done
 
 echo "Creating postgres database for Galaxy"
 createdb -w -U postgres -h postgres galaxy

.ci/jenkins/selenium/run_tests.sh

@@ -36,6 +36,12 @@ docker-compose down | true
 docker-compose build galaxy
 docker-compose up -d
 
+function tear_down {
+    docker-compose down
+}
+
+trap tear_down EXIT
+
 for service_name in postgres galaxy selenium
 do
     echo "Waiting on service ${service_name}"
@@ -79,7 +85,6 @@ do
                 docker logs "${container_id}"
                 echo "---"
             done
-            docker-compose down | true
             exit 1
         fi
     done
@@ -92,6 +97,12 @@ done;
 export GALAXY_TEST_SELENIUM_REMOTE=1
 export GALAXY_TEST_SELENIUM_REMOTE_PORT="${SELENIUM_PORT}"
 
+# Retry all failed Selenium tests a second time to deal
+# with transiently failing tests. Failure information for
+# first tests is still populated in database/test_errors
+# and available at the top of the Jenkins test report.
+export GALAXY_TEST_SELENIUM_RETRIES=1
+
 # Access Galaxy on localhost via port $GALAXY_PORT
 export GALAXY_TEST_PORT="${GALAXY_PORT}"
 
@@ -105,6 +116,4 @@ exit_code=$?
 
 cd $TEST_DIRECTORY
 
-docker-compose down
-
 exit $exit_code

.ci/py3_sources.txt

@@ -6,69 +6,36 @@ doc/patch.py
 lib/galaxy/actions/
 lib/galaxy/auth/
 lib/galaxy/config.py
+lib/galaxy/containers/
 lib/galaxy/dataset_collections/
-lib/galaxy/datatypes/assembly.py
-lib/galaxy/datatypes/binary.py
-lib/galaxy/datatypes/constructive_solid_geometry.py
-lib/galaxy/datatypes/converters/
-lib/galaxy/datatypes/dataproviders/
-lib/galaxy/datatypes/data.py
-lib/galaxy/datatypes/images.py
-lib/galaxy/datatypes/msa.py
-lib/galaxy/datatypes/ngsindex.py
-lib/galaxy/datatypes/proteomics.py
-lib/galaxy/datatypes/sequence.py
-lib/galaxy/datatypes/sniff.py
-lib/galaxy/datatypes/tabular.py
-lib/galaxy/datatypes/tracks.py
+lib/galaxy/datatypes/
 lib/galaxy/dependencies/
 lib/galaxy/eggs/
 lib/galaxy/exceptions/
 lib/galaxy/external_services/
 lib/galaxy/forms/
 lib/galaxy/jobs/
 lib/galaxy/managers/
-lib/galaxy/model/__init__.py
-lib/galaxy/model/item_attrs.py
-lib/galaxy/model/mapping.py
-lib/galaxy/model/metadata.py
-lib/galaxy/model/migrate/
-lib/galaxy/model/orm/now.py
+lib/galaxy/model/
 lib/galaxy/objectstore/
 lib/galaxy/openid/
 lib/galaxy/quota/
 lib/galaxy/sample_tracking/
 lib/galaxy/security/
 lib/galaxy/tags/
-lib/galaxy/tools/actions/
-lib/galaxy/tools/cwl/
-lib/galaxy/tools/deps/
-lib/galaxy/tools/exception_handling.py
-lib/galaxy/tools/execute.py
-lib/galaxy/tools/lint.py
-lib/galaxy/tools/lint_util.py
-lib/galaxy/tools/linters/
-lib/galaxy/tools/loader.py
-lib/galaxy/tools/loader_directory.py
-lib/galaxy/tools/parameters/dataset_matcher.py
-lib/galaxy/tools/parameters/__init__.py
-lib/galaxy/tools/parameters/input_translation.py
-lib/galaxy/tools/parameters/sanitize.py
-lib/galaxy/tools/parameters/validation.py
-lib/galaxy/tools/parameters/wrapped_json.py
-lib/galaxy/tools/parameters/wrapped.py
-lib/galaxy/tools/parser/
-lib/galaxy/tools/test.py
-lib/galaxy/tools/toolbox/
+lib/galaxy/tools/
 lib/galaxy/tours/
 lib/galaxy/util/
 lib/galaxy/visualization/
 lib/galaxy/web/buildapp.py
+lib/galaxy/web/framework/base.py
+lib/galaxy/web/framework/decorators.py
 lib/galaxy/web/framework/__init__.py
 lib/galaxy/web/framework/middleware/error.py
 lib/galaxy/web/framework/middleware/static.py
 lib/galaxy/web/framework/middleware/statsd.py
 lib/galaxy/web/__init__.py
+lib/galaxy/webapps/galaxy/api/histories.py
 lib/galaxy/webapps/galaxy/api/tours.py
 lib/galaxy/webapps/galaxy/api/workflows.py
 lib/galaxy/webapps/galaxy/controllers/external_services.py
@@ -81,13 +48,10 @@ lib/galaxy/webapps/tool_shed/controllers/user.py
 lib/galaxy/webapps/tool_shed/framework/middleware/remoteuser.py
 lib/galaxy/webapps/tool_shed/__init__.py
 lib/galaxy/webapps/tool_shed/util/ratings_util.py
+lib/galaxy/webhooks/
 lib/galaxy/work/
-lib/galaxy/workflow/extract.py
-lib/galaxy/workflow/run.py
-lib/galaxy/workflow/schedulers/core.py
-lib/galaxy/workflow/steps.py
+lib/galaxy/workflow/
 lib/galaxy_ext/
-lib/galaxy_utils/
 lib/log_tempfile.py
 lib/psyco_full.py
 lib/tool_shed/

.gitignore

@@ -17,6 +17,7 @@ database/files
 database/jobs_directory
 database/job_working_directory
 database/pbs
+database/test_errors
 database/tmp
 database/*.sqlite
 database/openid_consumer_cache
@@ -93,7 +94,7 @@ tool-data/genome/*
 tool-data/*.sample
 tool-data/testtoolshed.g2.bx.psu.edu/
 tool-data/toolshed.g2.bx.psu.edu/
-tool-data/**/*.fa 
+tool-data/**/*.fa
 
 # Test output
 test-data-cache
@@ -121,7 +122,7 @@ tool-data/shared/jars/
 sprite-*.less
 
 # JS, Local node_modules, and bower_components directories
-static/scripts/bundled
+static/scripts/bundled/[0-9]*
 node_modules
 bower_components
 
@@ -135,4 +136,4 @@ doc/source/dev/schema.rst
 .DS_Store
 *.rej
 *~
-.idea/
+.idea

.travis.yml

@@ -1,3 +1,4 @@
+sudo: false
 language: python
 python: 2.7
 os:
@@ -24,9 +25,13 @@ matrix:
   allow_failures:
   - env: TOX_ENV=py27-lint-imports
 
+addons:
+  apt:
+    packages:
+      - libxml2-utils
+
 install:
   - pip install tox
-  - if [ "$TOX_ENV" == "validate-test-tools" ]; then sudo apt-get install libxml2-utils; fi
   - if [ "$TOX_ENV" == "qunit" ]; then bash -c 'cd test/qunit && npm install'; fi
   - if [ "$TOX_ENV" == "first_startup" ]; then bash -c "bash scripts/common_startup.sh && wget -q https://github.com/jmchilton/galaxy-downloads/raw/master/db_gx_rev_0127.sqlite && mv db_gx_rev_0127.sqlite database/universe.sqlite && bash manage_db.sh -c ./config/galaxy.ini.sample upgrade"; fi
 

CITATION

@@ -16,7 +16,7 @@ following publications:
   a platform for interactive large-scale genome analysis." 
   Genome Research. 2005 Oct; 15(10):1451-5.
 
-See also: http://wiki.galaxyproject.org/CitingGalaxy
+See also: https://galaxyproject.org/citing-galaxy
 
 
 

CONTRIBUTING.md

@@ -6,26 +6,25 @@ galaxy project](https://github.com/galaxyproject/galaxy) -
 also checkout our 2013 Galaxy Community
 Conference presentation on the topic
 ([video](https://vimeo.com/channels/581875/73486255),
-[presentation](https://wiki.galaxyproject.org/Documents/Presentations/GCC2013?action=AttachFile&do=view&target=BakerContribute.pdf)). For
+[presentation](https://depot.galaxyproject.org/hub/attachments/documents/presentations/gcc2013/BakerContribute.pdf)). For
 information on contributing more broadly to the Galaxy ecosystem and a
 deeper discussion of some of these points - please see the
-[Develop](https://wiki.galaxyproject.org/Develop/) section of the
-[Galaxy Wiki](https://wiki.galaxyproject.org/).
+[Develop](https://galaxyproject.org/develop) section of the
+[Galaxy Community Hub](https://galaxyproject.org).
 
 ## Before you Begin
 
 If you have an idea for a feature to add or an approach for a bugfix,
 it is best to communicate with Galaxy developers early. The most
 common venues for this are
-[GitHub issues](https://github.com/galaxyproject/galaxy/issues) and the
-[Galaxy and Tool Shed Trello boards](https://wiki.galaxyproject.org/Issues).
+[GitHub issues](https://github.com/galaxyproject/galaxy/issues).
 Browse through existing GitHub issues and Trello cards and if one seems related,
-comment on it. We also maintain a [card](https://trello.com/c/eFdPIdIB) with
-links to smaller issues we believe would make the best entry points for new
+comment on it. We also maintain a [tag](https://github.com/galaxyproject/galaxy/issues?q=is%3Aissue+is%3Aopen+label%3Afriendliness%2Ffriendly) on Github for
+smaller issues we believe would make the best entry points for new
 developers.
 Galaxy developers are generally available via
-[IRC](https://wiki.galaxyproject.org/GetInvolved#IRC_Channel) and on
-the [development mailing list](http://dev.list.galaxyproject.org/).
+[IRC](https://galaxyproject.org/get-involved), [Gitter](https://gitter.im/galaxyproject/Lobby)
+and on the [development mailing list](http://dev.list.galaxyproject.org).
 
 ## Reporting a new issue
 
@@ -67,21 +66,18 @@ opened using [this form](https://github.com/galaxyproject/galaxy/issues/new).
     should be made against the recent `release_XX.XX` branch (`git checkout release_XX.XX`).
 
   * Serious security problems should not be fixed via pull request - please
-    responsibly disclose these by e-mailing them (with or without patches) to
-    [email protected] . The Galaxy core development team will
-    issue patches to public servers before announcing the issue to ensure there
-    is time to patch and highlight these fixes widely. We will provide you
-    credit for the discovery when publicly disclosing the issue.
+    see [the Galaxy security policies](SECURITY_POLICY.md) for information
+    about responsibly disclosing security issues.
 
 * If your changes modify code - please ensure the resulting files
   conform to Galaxy [style
-  guidelines](https://wiki.galaxyproject.org/Develop/BestPractices).
+  guidelines](https://galaxyproject.org/develop/best-practices).
 
 * Galaxy contains hundreds of tests of different types and complexity
   and running each is difficult and probably not reasonable at this
   time (someday we will provide a holistic test procedure to make this
   possible). For now, please just review the [running tests
-  documentation](https://wiki.galaxyproject.org/Admin/RunningTests)
+  documentation](https://galaxyproject.org/admin/running-tests)
   and run any that seem relevant. Developers reviewing your pull
   request will be happy to help guide you to running the most relevant
   tests as part of the pull request review process and may request the
@@ -110,11 +106,10 @@ opened using [this form](https://github.com/galaxyproject/galaxy/issues/new).
 ## A Quick Note about Tools
 
   For the most part, Galaxy tools should be published to the
-  [Tool Shed](https://wiki.galaxyproject.org/ToolShed) and not in this
-  repository directly. If you are looking to supply fixes for migrated
+  [Tool Shed](https://galaxyproject.org/toolshed) and not in this
+  repository directly. If you are looking for migrated
   core tools that used to exist in this repository - please checkout
-  the [tools-devteam](https://github.com/galaxyproject/tools-devteam)
+  the [tools-iuc](https://github.com/galaxyproject/tools-iuc)
   repository on GitHub.
 
-  More information about tool development can be found [on the
-  wiki](https://wiki.galaxyproject.org/Develop).
+  More information about tool development can be found [on the comunity hub](https://galaxyproject.org/develop).