Merge pull request #1224 from nsoranzo/update_best_practice_search

Update `best_practice_search()` for changes in galaxy-tool-util

planemo/conda.py

@@ -8,6 +8,7 @@
 import collections
 import os
 import threading
+from copy import deepcopy
 
 from galaxy.tool_util.deps import conda_util
 from galaxy.util import unicodify
@@ -154,12 +155,15 @@ def best_practice_search(conda_target, conda_context=None, platform=None):
         best_practice_search_first.previously_called = True
         offline = False
 
-    if not conda_context:
-        conda_context = conda_util.CondaContext()
+    if conda_context:
+        if conda_context.ensure_channels != BEST_PRACTICE_CHANNELS:
+            conda_context = deepcopy(conda_context)
+            conda_context.ensure_channels = BEST_PRACTICE_CHANNELS
+    else:
+        conda_context = conda_util.CondaContext(ensure_channels=BEST_PRACTICE_CHANNELS)
     return conda_util.best_search_result(
         conda_target,
         conda_context=conda_context,
-        channels_override=BEST_PRACTICE_CHANNELS,
         offline=offline,
         platform=platform,
     )

planemo/database/factory.py

@@ -1,5 +1,5 @@
 """Create a DatabaseSource from supplied planemo configuration."""
-from galaxy.tool_util.deps.commands import which
+from galaxy.util.commands import which
 
 from .postgres import LocalPostgresDatabaseSource
 from .postgres_docker import DockerPostgresDatabaseSource

planemo/database/postgres_docker.py

@@ -5,8 +5,8 @@
     docker_util,
     dockerfiles,
 )
-from galaxy.tool_util.deps.commands import execute
 from galaxy.util import unicodify
+from galaxy.util.commands import execute
 
 from .interface import DatabaseSource
 from .postgres import _CommandBuilder, ExecutesPostgresSqlMixin

planemo/galaxy/run.py

@@ -2,7 +2,7 @@
 import os
 import string
 
-from galaxy.tool_util.deps.commands import shell
+from galaxy.util.commands import shell
 from six.moves import shlex_quote
 
 from planemo.io import info, shell_join

planemo/xml/validation.py

@@ -3,8 +3,8 @@
 import subprocess
 from collections import namedtuple
 
-from galaxy.tool_util.deps.commands import which
 from galaxy.util import unicodify
+from galaxy.util.commands import which
 try:
     from lxml import etree
 except ImportError:

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+markers =
+    tests_galaxy_branch: test against a specific Galaxy branch

tests/shed_app.py

@@ -47,7 +47,7 @@ def update_repository(id):
 def update_repository_contents(id):
     updated_tar = request.files['file']
     message = request.form["commit_message"]
-    repo_path = app.config["model"].repository_path_for_update(id, message)
+    repo_path = app.config["model"].repository_path(id)
     repo_tar_path = repo_path + ".tar.gz"
     if not os.path.exists(repo_path):
         os.makedirs(repo_path)
@@ -58,7 +58,7 @@ def update_repository_contents(id):
     finally:
         tar.close()
     _modify_repository(repo_path)
-    return json.dumps({"id": id})
+    return json.dumps({"message": message})
 
 
 @app.route("/api/categories")
@@ -94,23 +94,15 @@ def repository_download():
     return send_file(repo_tar_download_path)
 
 
-@app.route('/shutdown', methods=['POST'])
-def shutdown():
-    # Used to shutdown test server.
-    _shutdown_server()
-    return ''
-
-
 def _request_post_message():
     return json.loads(request.data.decode("utf-8"))
 
 
-class InMemoryShedDataModel(object):
+class InMemoryShedDataModel:
 
     def __init__(self, directory):
         self.directory = directory
         self._repositories = {}
-        self._repositories_msg = {}
         self._categories = []
 
     def add_category(self, id, name):
@@ -141,12 +133,6 @@ def get_repository(self, id):
     def repository_path(self, id):
         return os.path.join(self.directory, id)
 
-    def repository_path_for_update(self, id, message):
-        if id not in self._repositories_msg:
-            self._repositories_msg[id] = []
-        self._repositories_msg[id].append(message)
-        return self.repository_path(id)
-
 
 def _modify_repository(path):
     arch = os.path.join(path, ".hg_archival.txt")
@@ -175,10 +161,3 @@ def _modify_attributes(xml_element):
 
     for child in xml_element:
         _modify_attributes(child)
-
-
-def _shutdown_server():
-    func = request.environ.get('werkzeug.server.shutdown')
-    if func is None:
-        raise RuntimeError('Not running with the Werkzeug Server')
-    func()

tests/shed_app_test_utils.py

@@ -1,10 +1,9 @@
 import contextlib
+import multiprocessing
 import shutil
-import threading
-from collections import namedtuple
 from tempfile import mkdtemp
+from typing import NamedTuple
 
-from requests import post
 from werkzeug.serving import run_simple
 
 from planemo import network_util
@@ -48,10 +47,10 @@ def run():
             use_debugger=True
         )
 
-    t = threading.Thread(target=run)
-    t.start()
+    p = multiprocessing.Process(target=run)
+    p.start()
     network_util.wait_net_service("localhost", port, DEFAULT_OP_TIMEOUT)
-    return MockShed("http://localhost:%d" % port, directory, t, model)
+    return MockShed(f"http://localhost:{port}", directory, p, model)
 
 
 @contextlib.contextmanager
@@ -65,14 +64,16 @@ def mock_shed():
             mock_shed_obj.shutdown()
 
 
-def _shutdown(self):
-    post("%s/shutdown" % self.url)
-    self.thread.join(DEFAULT_OP_TIMEOUT)
-    shutil.rmtree(self.directory)
+class MockShed(NamedTuple):
+    url: str
+    directory: str
+    process: multiprocessing.Process
+    model: InMemoryShedDataModel
 
+    def shutdown(self):
+        self.process.terminate()
+        shutil.rmtree(self.directory)
 
-MockShed = namedtuple("MockShed", ["url", "directory", "thread", "model"])
-MockShed.shutdown = _shutdown
 
 __all__ = (
     "setup_mock_shed",

tests/test_engines.py

@@ -8,7 +8,7 @@
     get_outputs,
     RunnableType,
 )
-from .test_utils import test_context, TEST_DATA_DIR
+from .test_utils import create_test_context, TEST_DATA_DIR
 
 A_CWL_TOOL = os.path.join(TEST_DATA_DIR, "tools", "ok-cat1-tool.cwl")
 A_CWL_WORKFLOW = os.path.join(TEST_DATA_DIR, "count-lines2-wf.cwl")
@@ -36,7 +36,7 @@
 
 
 def test_can_handle():
-    ctx = test_context()
+    ctx = create_test_context()
     for engine_type in ["galaxy", "cwltool"]:
         with engine_context(ctx, engine=engine_type) as e:
             for key, value in CAN_HANDLE[engine_type].items():

tests/test_galaxy_config.py

@@ -9,9 +9,9 @@
     get_refgenie_config,
 )
 from .test_utils import (
+    create_test_context,
     skip_if_environ,
     TempDirectoryContext,
-    test_context,
 )
 
 
@@ -59,7 +59,7 @@ def _assert_property_is(config, prop, value):
 
 @contextlib.contextmanager
 def _test_galaxy_config(tool_paths=[], **kwargs):
-    ctx = test_context()
+    ctx = create_test_context()
     with TempDirectoryContext() as tdc:
         test_data = os.path.join(tdc.temp_directory, "test-data")
         os.makedirs(test_data)

tests/test_galaxy_test.py

@@ -7,8 +7,8 @@
 from planemo.galaxy.test.actions import passed
 from planemo.galaxy.test.actions import run_in_config
 from .test_utils import (
+    create_test_context,
     TempDirectoryTestCase,
-    test_context,
     TEST_DATA_DIR,
 )
 
@@ -25,7 +25,7 @@ def setUp(self):
         """Setup mock keywords, context, and Galaxy config for tests."""
         super(RunInConfigTestCase, self).setUp()
         td = self.temp_directory
-        self.ctx = test_context()
+        self.ctx = create_test_context()
         self.config = _MockConfig(td)
         self.kwds = {
             "test_output": os.path.join(td, "tests.html"),

tests/test_github_create_release.py

@@ -12,7 +12,7 @@
 )
 from planemo.io import temp_directory
 from .test_utils import (
-    test_context,
+    create_test_context,
 )
 
 
@@ -44,14 +44,14 @@ def test_changelog_in_repo():
 
 
 def test_get_or_create_repo_with_existing_repo():
-    ctx = test_context()
+    ctx = create_test_context()
     ctx._global_config = {"github": {"access_token": 'ABCDEFG'}}
     repository_path = get_or_create_repository(ctx, owner='galaxyproject', repo='planemo', dry_run=False)
     assert os.path.exists(os.path.join(repository_path, 'README.rst'))
 
 
 def test_get_or_create_repo_with_new_repo():
-    ctx = test_context()
+    ctx = create_test_context()
     ctx._global_config = {"github": {"access_token": 'ABCDEFG'}}
     with pytest.raises(RuntimeError) as excinfo:
         # Token isn't valid, so this errors out while running gh create
@@ -61,7 +61,7 @@ def test_get_or_create_repo_with_new_repo():
 
 
 def test_add_dir_contents_to_repo():
-    ctx = test_context()
+    ctx = create_test_context()
     ctx._global_config = {"github": {"access_token": 'ABCDEFG'}}
     with temp_directory() as test_dir, temp_directory() as repo_dir:
         with open(os.path.join(test_dir, 'Readme.md'), 'w') as readme:
@@ -82,7 +82,7 @@ def test_add_dir_contents_to_repo():
 
 
 def test_add_dir_contents_to_repo_dry_run():
-    ctx = test_context()
+    ctx = create_test_context()
     ctx._global_config = {"github": {"access_token": 'ABCDEFG'}}
     with temp_directory() as test_dir, temp_directory() as repo_dir:
         with open(os.path.join(test_dir, 'Readme.md'), 'w') as readme:
@@ -100,7 +100,7 @@ def test_add_dir_contents_to_repo_dry_run():
 
 
 def test_git_ls_remote():
-    ctx = test_context()
+    ctx = create_test_context()
     tags_and_commits = git.ls_remote(ctx, 'https://github.com/galaxyproject/galaxy')
     assert 'refs/heads/release_20.09' in tags_and_commits
 

tests/test_shed_create.py

@@ -16,7 +16,7 @@ def test_create_single(self):
             create_command.extend(self._shed_args())
             self._check_exit_code(create_command)
 
-    @skip
+    @skip("Broken, see https://github.com/galaxyproject/planemo/issues/437")
     def test_create_wrong_owner(self):
         with self._isolate_repo("single_tool_other_owner"):
             create_command = ["shed_create", "--skip_upload"]

tests/test_shed_upload.py

@@ -11,7 +11,6 @@
 
 from galaxy.util import unicodify
 
-from planemo import git
 from planemo.io import shell
 from .test_utils import (
     assert_exists,
@@ -144,20 +143,13 @@ def test_upload_from_git(self):
                     "git add .",
                     "git commit -m 'initial commit'"
                 ]))
-                rev = git.rev(None, "single_tool")
                 upload_command = [
                     "shed_update", "--force_repository_creation",
                     "git+single_tool/.git"
                 ]
                 upload_command.extend(self._shed_args())
                 self._check_exit_code(upload_command)
                 self._verify_single_uploaded(f, ["single_tool"])
-                model = self.mock_shed.model
-                repo_id = self.repository_by_name("single_tool")["id"]
-                message = model._repositories_msg[repo_id][0]
-                assert "planemo upload for repository " in message
-                assert "repository https://github.com/galaxyproject" in message
-                assert rev in message
 
     @contextlib.contextmanager
     def _git_configured(self):
@@ -248,7 +240,7 @@ def test_tar_with_symlinks(self):
             upload_command.extend(self._shed_args())
             self._check_exit_code(upload_command)
             target = self._verify_upload(f, ["macros.xml"], ["cat2"])
-            with open(join(target, "macros.xml"), "r") as macro_f:
+            with open(join(target, "macros.xml")) as macro_f:
                 macro_contents = macro_f.read()
                 assert macro_contents.startswith("<macros>")
 
@@ -375,22 +367,25 @@ def _verify_expansion(self, f, name=None):
             not_contains=["cat1.xml"]
         )
 
-    def _verify_single_uploaded(self, f, download_args=[]):
+    def _verify_single_uploaded(self, f, download_args=None):
         self._verify_upload(
             f, ["cat.xml", "related_file", "test-data/1.bed"], download_args
         )
 
-    def _verify_empty_repository(self, f, download_args=[]):
+    def _verify_empty_repository(self, f, download_args=None):
         target = self._download_repo(f, download_args)
         assert len(os.listdir(target)) == 0
 
-    def _verify_upload(self, f, download_files=[], download_args=[]):
+    def _verify_upload(self, f, download_files=None, download_args=None):
+        download_files = download_files or []
         target = self._download_repo(f, download_args)
         for download_file in download_files:
             assert_exists(join(target, download_file))
         return target
 
-    def _check_tar(self, f, tar_path, contains=[], not_contains=[]):
+    def _check_tar(self, f, tar_path, contains=None, not_contains=None):
+        contains = contains or []
+        not_contains = not_contains or []
         tar_path = join(f, tar_path)
         assert_exists(tar_path)
         target = self._untar(f, tar_path)
@@ -400,9 +395,10 @@ def _check_tar(self, f, tar_path, contains=[], not_contains=[]):
             assert not exists(join(target, path))
         return target
 
-    def _download_repo(self, f, download_args=[]):
+    def _download_repo(self, f, download_args=None):
         download_command = ["shed_download"]
-        download_command.extend(download_args)
+        if download_args:
+            download_command.extend(download_args)
         download_command.extend(self._shed_args(read_only=True))
         self._check_exit_code(download_command)
         download = join(f, "shed_download.tar.gz")