Skip to content
Permalink

Comparing changes

This is a direct comparison between two commits made in this repository or its related repositories. View the default comparison for this range or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: galaxyproject/galaxy
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 89b920d5ee5f22de8000b60645bcd1a5b8adeb41
Choose a base ref
..
head repository: galaxyproject/galaxy
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 47e39b03a00678d093f6549069b8d9c8c4d9e1fb
Choose a head ref
1 change: 1 addition & 0 deletions doc/schema_template.md
Original file line number Diff line number Diff line change
@@ -25,6 +25,7 @@ $tag:tool|inputs|conditional|when://complexType[@name='ConditionalWhen']
$tag:tool|inputs|param://complexType[@name='Param']
$tag:tool|inputs|param|validator://complexType[@name='Validator']
$tag:tool|inputs|param|option://complexType[@name='ParamOption']
$tag:tool|inputs|param|conversion://complexType[@name='ParamConversion']
$tag:tool|inputs|param|options://complexType[@name='ParamOptions']
$tag:tool|inputs|param|options|column://complexType[@name='Column']
$tag:tool|inputs|param|options|filter://complexType[@name='Filter']
2 changes: 1 addition & 1 deletion lib/galaxy/jobs/runners/condor.py
Original file line number Diff line number Diff line change
@@ -66,7 +66,7 @@ def queue_job( self, job_wrapper ):
container = None
universe = query_params.get('universe', None)
if universe and universe.strip().lower() == 'docker':
container = self.find_container( job_wrapper )
container = self._find_container( job_wrapper )
if container:
# HTCondor needs the image as 'docker_image'
query_params.update({'docker_image': container})
12 changes: 6 additions & 6 deletions lib/galaxy/tools/__init__.py
Original file line number Diff line number Diff line change
@@ -20,6 +20,8 @@

import galaxy.jobs
import tool_shed.util.repository_util as repository_util
import tool_shed.util.shed_util_common

from galaxy import (
exceptions,
model
@@ -83,7 +85,6 @@
from galaxy.web.form_builder import SelectField
from galaxy.work.context import WorkRequestContext
from tool_shed.util import common_util
from tool_shed.util import shed_util_common as suc

from .execute import execute as execute_job
from .loader import (
@@ -196,8 +197,8 @@ def has_reloaded(self, other_toolbox):

@property
def all_requirements(self):
reqs = [json.dumps(req, sort_keys=True) for _, tool in self.tools() for req in tool.tool_requirements]
return [json.loads(req) for req in set(reqs)]
reqs = set([req for _, tool in self.tools() for req in tool.tool_requirements])
return [r.to_dict() for r in reqs]

@property
def tools_by_id( self ):
@@ -1071,7 +1072,7 @@ def __inititalize_help(self):
if self.repository_id and help_text.find( '.. image:: ' ) >= 0:
# Handle tool help image display for tools that are contained in repositories in the tool shed or installed into Galaxy.
try:
help_text = suc.set_image_paths( self.app, self.repository_id, help_text )
help_text = tool_shed.util.shed_util_common.set_image_paths( self.app, self.repository_id, help_text )
except Exception as e:
log.exception( "Exception in parse_help, so images may not be properly displayed:\n%s" % str( e ) )
try:
@@ -1425,8 +1426,7 @@ def tool_requirements(self):
"""
Return all requiremens of type package
"""
reqs = [req for req in self.requirements if req.type == 'package']
return reqs
return self.requirements.packages

@property
def tool_requirements_status(self):
32 changes: 15 additions & 17 deletions lib/galaxy/tools/deps/__init__.py
Original file line number Diff line number Diff line change
@@ -14,7 +14,10 @@
plugin_config
)

from .requirements import ToolRequirement
from .requirements import (
ToolRequirement,
ToolRequirements
)
from .resolvers import NullDependency
from .resolvers.conda import CondaDependencyResolver
from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver
@@ -124,12 +127,7 @@ def _requirements_to_dependencies_dict(self, requirements, **kwds):
require_exact = kwds.get('exact', False)
return_null_dependencies = kwds.get('return_null', False)

resolvable_requirements = []
for requirement in requirements:
if requirement.type in ['package', 'set_environment']:
resolvable_requirements.append(requirement)
else:
log.debug("Unresolvable requirement type [%s] found, will be ignored." % requirement.type)
resolvable_requirements = requirements.resolvable

for i, resolver in enumerate(self.dependency_resolvers):
if index is not None and i != index:
@@ -157,16 +155,16 @@ def _requirements_to_dependencies_dict(self, requirements, **kwds):
if requirement in requirement_to_dependency:
continue

if requirement.type in ['package', 'set_environment']:
dependency = resolver.resolve( requirement, **kwds )
if require_exact and not dependency.exact:
continue
dependency = resolver.resolve( requirement, **kwds )
if require_exact and not dependency.exact:
continue

if not isinstance(dependency, NullDependency):
log.debug(dependency.resolver_msg)
if not isinstance(dependency, NullDependency):
requirement_to_dependency[requirement] = dependency
elif return_null_dependencies and (resolver == self.dependency_resolvers[-1] or i == index):
requirement_to_dependency[requirement] = dependency
requirement_to_dependency[requirement] = dependency
elif return_null_dependencies and (resolver == self.dependency_resolvers[-1] or i == index):
log.debug(dependency.resolver_msg)
requirement_to_dependency[requirement] = dependency

return requirement_to_dependency

@@ -175,8 +173,8 @@ def uses_tool_shed_dependencies(self):

def find_dep( self, name, version=None, type='package', **kwds ):
log.debug('Find dependency %s version %s' % (name, version))
requirement = ToolRequirement(name=name, version=version, type=type)
dep_dict = self._requirements_to_dependencies_dict([requirement], **kwds)
requirements = ToolRequirements([ToolRequirement(name=name, version=version, type=type)])
dep_dict = self._requirements_to_dependencies_dict(requirements, **kwds)
if len(dep_dict) > 0:
return dep_dict.values()[0]
else:
4 changes: 2 additions & 2 deletions lib/galaxy/tools/deps/dependencies.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from galaxy.tools.deps.requirements import ToolRequirement
from galaxy.tools.deps.requirements import ToolRequirements
from galaxy.util import bunch


@@ -29,7 +29,7 @@ def from_dict(as_dict):
return None

requirements_dicts = as_dict.get('requirements', [])
requirements = [ToolRequirement.from_dict(r) for r in requirements_dicts]
requirements = ToolRequirements.from_list(requirements_dicts)
installed_tool_dependencies_dicts = as_dict.get('installed_tool_dependencies', [])
installed_tool_dependencies = map(DependenciesDescription._toolshed_install_dependency_from_dict, installed_tool_dependencies_dicts)
return DependenciesDescription(
79 changes: 76 additions & 3 deletions lib/galaxy/tools/deps/requirements.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
import copy

from galaxy.util import asbool, xml_text
from galaxy.util import (
asbool,
xml_text,
)
from galaxy.util.oset import OrderedSet


DEFAULT_REQUIREMENT_TYPE = "package"
DEFAULT_REQUIREMENT_VERSION = None
@@ -33,6 +38,15 @@ def from_dict( dict ):
specs = [RequirementSpecification.from_dict(s) for s in dict.get("specs", [])]
return ToolRequirement( name=name, type=type, version=version, specs=specs )

def __eq__(self, other):
return self.name == other.name and self.type == other.type and self.version == other.version

def __ne__(self, other):
return not self.__eq__(other)

def __hash__(self):
return hash((self.name, self.type, self.version))


class RequirementSpecification(object):
"""Refine a requirement using a URI."""
@@ -61,10 +75,69 @@ def from_dict(dict):
def __eq__(self, other):
return self.name == other.name and self.type == other.type and self.version == other.version

def __ne__(self, other):
return not self.__eq__(other)

def __hash__(self):
return hash((self.name, self.type, self.version))


class ToolRequirements(object):
"""
Represents all requirements (packages, env vars) needed to run a tool.
"""
def __init__(self, tool_requirements=None):
if tool_requirements:
if not isinstance(tool_requirements, list):
raise ToolRequirementsException('ToolRequirements Constructor expects a list')
self.tool_requirements = OrderedSet([r if isinstance(r, ToolRequirement) else ToolRequirement.from_dict(r) for r in tool_requirements])
else:
self.tool_requirements = OrderedSet()

@staticmethod
def from_list(requirements):
return ToolRequirements(requirements)

@property
def resolvable(self):
return ToolRequirements([r for r in self.tool_requirements if r.type in {'package', 'set_environment'}])

@property
def packages(self):
return ToolRequirements([r for r in self.tool_requirements if r.type == 'package'])

def to_list(self):
return [r.to_dict() for r in self.tool_requirements]

def append(self, requirement):
if not isinstance(requirement, ToolRequirement):
requirement = ToolRequirement.from_dict(requirement)
self.tool_requirements.add(requirement)

def __eq__(self, other):
return len(self.tool_requirements & other.tool_requirements) == len(self.tool_requirements) == len(other.tool_requirements)

def __ne__(self, other):
return not self.__eq__(other)

def __iter__(self):
for r in self.tool_requirements:
yield r

def __getitem__(self, ii):
return list(self.tool_requirements)[ii]

def __len__(self):
return len(self.tool_requirements)

def __hash__(self):
return sum([r.__hash__() for r in self.tool_requirements])


class ToolRequirementsException(Exception):
pass


DEFAULT_CONTAINER_TYPE = "docker"
DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES = False
DEFAULT_CONTAINER_SHELL = "/bin/sh" # Galaxy assumes bash, but containers are usually thinner.
@@ -109,7 +182,7 @@ def from_dict( dict ):
def parse_requirements_from_dict( root_dict ):
requirements = root_dict.get("requirements", [])
containers = root_dict.get("containers", [])
return map(ToolRequirement.from_dict, requirements), map(ContainerDescription.from_dict, containers)
return ToolRequirements.from_list(requirements), map(ContainerDescription.from_dict, containers)


def parse_requirements_from_xml( xml_root ):
@@ -141,7 +214,7 @@ def parse_requirements_from_xml( xml_root ):
if requirements_elem is not None:
requirement_elems = requirements_elem.findall( 'requirement' )

requirements = []
requirements = ToolRequirements()
for requirement_elem in requirement_elems:
name = xml_text( requirement_elem )
type = requirement_elem.get( "type", DEFAULT_REQUIREMENT_TYPE )
16 changes: 8 additions & 8 deletions lib/galaxy/tools/deps/views.py
Original file line number Diff line number Diff line change
@@ -45,14 +45,14 @@ def manager_dependency(self, **kwds):
def resolver_dependency(self, index, **kwds):
return self._dependency(**kwds)

def show_dependencies(self, requirements, installed_tool_dependencies=None):
def show_dependencies(self, tool_requirements_d, installed_tool_dependencies=None):
"""
Resolves dependencies to build a requirements status in the admin panel/API
"""
kwds = {'install': False,
'return_null': True,
'installed_tool_dependencies': installed_tool_dependencies}
dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(dependencies, **kwds) for tool, dependencies in requirements.items()}
dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(requirements, **kwds) for tool, requirements in tool_requirements_d.items()}
return dependencies_per_tool

def install_dependencies(self, requirements):
@@ -138,12 +138,12 @@ def installable_resolvers(self):
"""
return [index for index, resolver in enumerate(self._dependency_resolvers) if hasattr(resolver, "install_dependency") and not resolver.disabled ]

def get_requirements_status(self, requested_requirements, installed_tool_dependencies=None):
dependencies = self.show_dependencies(requested_requirements, installed_tool_dependencies)
# dependencies is a dict keyed on tool_ids, values are lists of ToolRequirements for that tool.
# we collapse requested requirements to single set,
# then use collapsed requirements to get resolved dependencies without duplicates.
flat_tool_requirements = set([r for requirement_list in requested_requirements.values() for r in requirement_list])
def get_requirements_status(self, tool_requirements_d, installed_tool_dependencies=None):
dependencies = self.show_dependencies(tool_requirements_d, installed_tool_dependencies)
# dependencies is a dict keyed on tool_ids, value is a ToolRequirements object for that tool.
# We use the union of resolvable ToolRequirements to get resolved dependencies without duplicates.
requirements = [r.resolvable for r in tool_requirements_d.values()]
flat_tool_requirements = set().union(*requirements)
flat_dependencies = []
for requirements_odict in dependencies.values():
for requirement in requirements_odict:
44 changes: 14 additions & 30 deletions lib/galaxy/tools/errors.py
Original file line number Diff line number Diff line change
@@ -59,41 +59,25 @@
"""

error_report_template_html = """
<html><head></head>
<html>
<body>
<style type="text/css">
tr:nth-child(even) {background-color: #f2f2f2}
table{margin: 1em;}
.mono{font-family: monospace;}
pre {
white-space: pre-wrap;
white-space: -moz-pre-wrap;
white-space: -pre-wrap;
white-space: -o-pre-wrap;
word-wrap: break-word;
background: #eee;
border:1px solid black;
padding:10px;
}
</style>
<h1>Galaxy Tool Error Report</h1>
<span class="sub"><i>from</i> <span class="mono"><a href="${host}">${host}</a></span>
<span class="sub"><i>from</i> <span style="font-family: monospace;"><a href="${host}">${host}</a></span>
<h3>Error Localization</h3>
<table>
<table style="margin:1em">
<tbody>
<tr><td>Dataset</td><td>${dataset_id} (${dataset_id_encoded})</td></tr>
<tr><td>History</td><td><a href="${history_view_link}">${history_id} (${history_id_encoded})</a></td></tr>
<tr style="background-color: #f2f2f2"><td>History</td><td><a href="${history_view_link}">${history_id} (${history_id_encoded})</a></td></tr>
<tr><td>Failed Job</td><td>${hid}: ${history_item_name} (${hda_id_encoded})</td></tr>
</tbody>
</table>
<h3>User Provided Information</h3>
The user <a href="mailto:${email_str}"><span class="mono">${email_str}</span></a> provided the following information:
The user <a href="mailto:${email_str}"><span style="font-family: monospace;">${email_str}</span></a> provided the following information:
<pre>
<pre style="white-space: pre-wrap;background: #eeeeee;border:1px solid black;padding:1em;">
${message}
</pre>
@@ -102,40 +86,40 @@
Job environment and execution information is available at the job <a href="${hda_show_params_link}">Info Page</a>.
<table>
<table style="margin:1em">
<tbody>
<tr><td>Job ID</td><td>${job_id} (${job_id_encoded})</td></tr>
<tr><td>Tool ID</td><td>${job_tool_id}</td></tr>
<tr style="background-color: #f2f2f2"><td>Tool ID</td><td>${job_tool_id}</td></tr>
<tr><td>Tool Version</td><td>${tool_version}</td></tr>
<tr><td>Job PID or DRM id</td><td>${job_runner_external_id}</td></tr>
<tr style="background-color: #f2f2f2"><td>Job PID or DRM id</td><td>${job_runner_external_id}</td></tr>
<tr><td>Job Tool Version</td><td>${job_tool_version}</td></tr>
</tbody>
</table>
<h3>Job Execution and Failure Information</h3>
<h4>Command Line</h4>
<pre>
<pre style="white-space: pre-wrap;background: #eeeeee;border:1px solid black;padding:1em;">
${job_command_line}
</pre>
<h4>stderr</h4>
<pre>
<pre style="white-space: pre-wrap;background: #eeeeee;border:1px solid black;padding:1em;">
${job_stderr}
</pre>
<h4>stdout</h4>
<pre>
<pre style="white-space: pre-wrap;background: #eeeeee;border:1px solid black;padding:1em;">
${job_stdout}
</pre>
<h4>Job Information</h4>
<pre>
<pre style="white-space: pre-wrap;background: #eeeeee;border:1px solid black;padding:1em;">
${job_info}
</pre>
<h4>Job Traceback</h4>
<pre>
<pre style="white-space: pre-wrap;background: #eeeeee;border:1px solid black;padding:1em;">
${job_traceback}
</pre>
Loading