Merge pull request #5 from galaxyproject/master

Update repo from upstream
galaxyproject · Dec 17, 2018 · aaba105 · aaba105
2 parents 24122b2 + 91a4a5a
commit aaba105
Show file tree

Hide file tree

Showing 309 changed files with 13,263 additions and 3,136 deletions.
diff --git a/.tt_skip b/.tt_skip
@@ -3,4 +3,3 @@ deprecated/tools/htseq
 deprecated/tools/rglasso
 deprecated/tools/tool_factory_2
 deprecated/tools/differential_count_models
-tools/gemini
diff --git a/...gers/data_manager_gemini_database_downloader/data_manager/data_manager_gemini_download.py b/...gers/data_manager_gemini_database_downloader/data_manager/data_manager_gemini_download.py
@@ -6,18 +6,51 @@
 import subprocess
 import sys
 
+import yaml
+
 
 def main():
     today = datetime.date.today()
     params = json.loads( open( sys.argv[1] ).read() )
     target_directory = params[ 'output_data' ][0]['extra_files_path']
     os.mkdir( target_directory )
-    cmd = "gemini --annotation-dir %s update --dataonly %s %s" % (target_directory, params['param_dict']['gerp_bp'], params['param_dict']['cadd'] )
-    subprocess.check_call( cmd, shell=True )
+    # The target_directory needs to be specified twice for the following
+    # invocation of gemini.
+    # In essence, the GEMINI_CONFIG environment variable makes gemini store
+    # its yaml configuration file in that directory, while the
+    # --annotation-dir argument makes it write the same path into the yaml
+    # file, which is then used for determining where the actual annotation
+    # files should be stored.
+    gemini_env = os.environ.copy()
+    gemini_env['GEMINI_CONFIG'] = target_directory
+    cmd = "gemini --annotation-dir %s update --dataonly %s %s" % (
+        target_directory,
+        params['param_dict']['gerp_bp'],
+        params['param_dict']['cadd']
+    )
+    subprocess.check_call( cmd, shell=True, env=gemini_env )
+
+    # modify the newly created gemini config file to contain a relative
+    # annotation dir path, which will be interpreted as relative to
+    # the job working directory at runtime by any gemini tool
+    config_file = os.path.join(target_directory, 'gemini-config.yaml')
+    with open(config_file) as fi:
+        config = yaml.load(fi)
+    config['annotation_dir'] = 'gemini/data'
+    with open(config_file, 'w') as fo:
+        yaml.dump(config, fo, allow_unicode=False, default_flow_style=False)
+
     data_manager_dict = {
         'data_tables': {
-            'gemini_databases': [
-                {'value': today.isoformat(), 'dbkey': 'hg19', 'name': 'GEMINI annotations (%s)' % today.isoformat(), 'path': './%s' % today.isoformat() }
+            'gemini_versioned_databases': [
+                {
+                    'value': today.isoformat(),
+                    'dbkey': 'hg19',
+                    'version': params['param_dict']['gemini_db_version'],
+                    'name':
+                        'GEMINI annotations (%s snapshot)' % today.isoformat(),
+                    'path': './%s' % today.isoformat()
+                }
             ]
         }
     }

diff --git a/...ers/data_manager_gemini_database_downloader/data_manager/data_manager_gemini_download.xml b/...ers/data_manager_gemini_database_downloader/data_manager/data_manager_gemini_download.xml
@@ -1,7 +1,11 @@
-<tool id="data_manager_gemini_download" name="GEMINI Download" version="0.19.1" tool_type="manage_data">
-    <description>Download a new database</description>
+<tool id="data_manager_gemini_download" name="GEMINI Download" version="@[email protected]" tool_type="manage_data">
+    <description>the annotation files required by the GEMINI suite of tools</description>
+    <macros>
+        <token name="@VERSION@">0.18.1</token>
+        <token name="@DB_VERSION@">181</token>
+    </macros>
     <requirements>
-        <requirement type="package" version="0.19.1">gemini</requirement>
+        <requirement type="package" version="@VERSION@">gemini</requirement>
     </requirements>
     <command detect_errors="exit_code">
         python '$__tool_directory__/data_manager_gemini_download.py' '$out_file'
@@ -11,16 +15,21 @@
             label="Download CADD scores for GEMINI database annotation" help="(--extra cadd_score)"/>
         <param name="gerp_bp" type="boolean" truevalue="--extra gerp_bp" falsevalue="" checked="True"
             label="Download GERP for GEMINI database annotation" help="(--extra gerp_bp)"/>
+        <param name="gemini_db_version" type="hidden" value="@DB_VERSION@"/>
     </inputs>
     <outputs>
         <data name="out_file" format="data_manager_json" label="${tool.name}"/>
     </outputs>
     <tests>
     </tests>
     <help>
-This tool downloads the GEMINI databases.
+This tool downloads the GEMINI annotation files and makes them available to
+corresponding versions of the GEMINI suite of tools. Downloading the (very large) CADD_ scores and GERP_ annotation files is optional.
 
-For details about this tool, please go to http://gemini.readthedocs.org
+Please visit http://gemini.readthedocs.io for details about GEMINI.
+
+.. _GERP: http://mendel.stanford.edu/SidowLab/downloads/gerp/index.html
+.. _CADD: https://cadd.gs.washington.edu/
     </help>
     <citations>
         <citation type="doi">10.1371/journal.pcbi.1003153</citation>

diff --git a/data_managers/data_manager_gemini_database_downloader/data_manager_conf.xml b/data_managers/data_manager_gemini_database_downloader/data_manager_conf.xml
@@ -1,16 +1,17 @@
 <?xml version="1.0"?>
 <data_managers>
     <data_manager tool_file="data_manager/data_manager_gemini_download.xml" id="data_manager_gemini_download" >
-        <data_table name="gemini_databases">  <!-- Defines a Data Table to be modified. -->
+        <data_table name="gemini_versioned_databases">  <!-- Defines a Data Table to be modified. -->
             <output> <!-- Handle the output of the Data Manager Tool -->
                 <column name="value" /> <!-- columns that are going to be specified by the Data Manager Tool -->
                 <column name="dbkey" /> <!-- columns that are going to be specified by the Data Manager Tool -->
+                <column name="version" /> <!-- columns that are going to be specified by the Data Manager Tool -->
                 <column name="name" />  <!-- columns that are going to be specified by the Data Manager Tool -->
                 <column name="path" output_ref="out_file" >
                     <move type="directory" relativize_symlinks="True">
-                        <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">gemini/data/${dbkey}/${value}</target>
+                        <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">gemini/${version}/${dbkey}/${value}</target>
                     </move>
-                    <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/gemini/data/${dbkey}/${value}/gemini/data/</value_translation>
+                    <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/gemini/${version}/${dbkey}/${value}/</value_translation>
                     <value_translation type="function">abspath</value_translation>
                 </column>
             </output>

diff --git a/data_managers/data_manager_gemini_database_downloader/tool-data/gemini_databases.loc.sample b/data_managers/data_manager_gemini_database_downloader/tool-data/gemini_databases.loc.sample
diff --git a/...s/data_manager_gemini_database_downloader/tool-data/gemini_versioned_databases.loc.sample b/...s/data_manager_gemini_database_downloader/tool-data/gemini_versioned_databases.loc.sample
@@ -0,0 +1,3 @@
+## GEMINI versioned databases
+#DownloadDate	dbkey	DBversion	Description
+#2018-07-08	hg19	181	GEMINI annotations (2018-07-08 snapshot)
diff --git a/data_managers/data_manager_gemini_database_downloader/tool_data_table_conf.xml.sample b/data_managers/data_manager_gemini_database_downloader/tool_data_table_conf.xml.sample
@@ -1,7 +1,7 @@
 <tables>
-    <table name="gemini_databases" comment_char="#" allow_duplicate_entries="False">
-        <columns>value, dbkey, name, path</columns>
-        <file path="tool-data/gemini_databases.loc" />
+    <table name="gemini_versioned_databases" comment_char="#" allow_duplicate_entries="False">
+        <columns>value, dbkey, version, name, path</columns>
+        <file path="tool-data/gemini_versioned_databases.loc" />
     </table>
 </tables>
 
diff --git a/macros/read_group_macros.xml b/macros/read_group_macros.xml
@@ -242,7 +242,7 @@
         <param name="PU" type="text" label="Platform unit (PU)" help="Unique identifier (e.g. flowcell-barcode.lane for Illumina or slide for SOLiD)" optional="True" />
     </xml>
     <xml name="read_group_pu_required_param">
-        <param name="PU" type="text" label="Platform unit (PU)" help="Unique identifier (e.g. flowcell-barcode.lane for Illumina or slide for SOLiD)" optional="False" />
+        <param name="PU" type="text" value="run" label="Platform unit (PU)" help="Unique identifier (e.g. flowcell-barcode.lane for Illumina or slide for SOLiD)" optional="False" />
     </xml>
     <!-- Only ID is required - all groups available -->
     <xml name="read_group_inputs_spec">

diff --git a/tools/bedtools/annotateBed.xml b/tools/bedtools/annotateBed.xml
@@ -1,4 +1,4 @@
-<tool id="bedtools_annotatebed" name="AnnotateBed" version="@WRAPPER_VERSION@.0">
+<tool id="bedtools_annotatebed" name="bedtools AnnotateBed" version="@WRAPPER_VERSION@">
     <description>annotate coverage of features from multiple files</description>
     <macros>
         <import>macros.xml</import>

diff --git a/tools/bedtools/bamToBed.xml b/tools/bedtools/bamToBed.xml
@@ -1,4 +1,4 @@
-<tool id="bedtools_bamtobed" name="BAM to BED" version="@WRAPPER_VERSION@.0">
+<tool id="bedtools_bamtobed" name="bedtools BAM to BED" version="@WRAPPER_VERSION@">
     <description>converter</description>
     <macros>
         <import>macros.xml</import>

diff --git a/tools/bedtools/bamToFastq.xml b/tools/bedtools/bamToFastq.xml
@@ -1,4 +1,4 @@
-<tool id="bedtools_bamtofastq" name="Convert from BAM to FastQ" version="@WRAPPER_VERSION@.0">
+<tool id="bedtools_bamtofastq" name="bedtools Convert from BAM to FastQ" version="@WRAPPER_VERSION@">
     <description></description>
     <macros>
         <import>macros.xml</import>
@@ -26,6 +26,12 @@
             <filter>fq2 is True</filter>
         </data>
     </outputs>
+    <tests>
+        <test>
+            <param name="input" value="srma_in3.bam" ftype="bam" />
+            <output name="output" file="bamToFastq_result1.fastq" ftype="fastq" />
+        </test>
+    </tests>
 <help>
 
 **What it does**

diff --git a/tools/bedtools/bed12ToBed6.xml b/tools/bedtools/bed12ToBed6.xml
@@ -1,4 +1,4 @@
-<tool id="bedtools_bed12tobed6" name="BED12 to BED6" version="@WRAPPER_VERSION@.0">
+<tool id="bedtools_bed12tobed6" name="bedtools BED12 to BED6" version="@WRAPPER_VERSION@">
     <description>converter</description>
     <macros>
         <import>macros.xml</import>

diff --git a/tools/bedtools/bedToBam.xml b/tools/bedtools/bedToBam.xml
@@ -1,4 +1,4 @@
-<tool id="bedtools_bedtobam" name="BED to BAM" version="@WRAPPER_VERSION@.0">
+<tool id="bedtools_bedtobam" name="bedtools BED to BAM" version="@WRAPPER_VERSION@">
     <description>converter</description>
     <macros>
         <import>macros.xml</import>

diff --git a/tools/bedtools/bedToIgv.xml b/tools/bedtools/bedToIgv.xml
@@ -1,4 +1,4 @@
-<tool id="bedtools_bedtoigv" name="BED to IGV" version="@WRAPPER_VERSION@.0">
+<tool id="bedtools_bedtoigv" name="bedtools BED to IGV" version="@WRAPPER_VERSION@">
     <description>create batch script for taking IGV screenshots</description>
     <macros>
         <import>macros.xml</import>

diff --git a/tools/bedtools/bedpeToBam.xml b/tools/bedtools/bedpeToBam.xml
@@ -1,4 +1,4 @@
-<tool id="bedtools_bedpetobam" name="BEDPE to BAM" version="@WRAPPER_VERSION@.0">
+<tool id="bedtools_bedpetobam" name="bedtools BEDPE to BAM" version="@WRAPPER_VERSION@">
     <description>converter</description>
     <macros>
         <import>macros.xml</import>

diff --git a/tools/bedtools/closestBed.xml b/tools/bedtools/closestBed.xml
@@ -1,4 +1,4 @@
-<tool id="bedtools_closestbed" name="ClosestBed" version="@WRAPPER_VERSION@.0">
+<tool id="bedtools_closestbed" name="bedtools ClosestBed" version="@WRAPPER_VERSION@">
     <description>find the closest, potentially non-overlapping interval</description>
     <macros>
         <import>macros.xml</import>

diff --git a/tools/bedtools/clusterBed.xml b/tools/bedtools/clusterBed.xml
@@ -1,4 +1,4 @@
-<tool id="bedtools_clusterbed" name="ClusterBed" version="@WRAPPER_VERSION@.0">
+<tool id="bedtools_clusterbed" name="bedtools ClusterBed" version="@WRAPPER_VERSION@">
     <description>cluster overlapping/nearby intervals</description>
     <macros>
         <import>macros.xml</import>

diff --git a/tools/bedtools/complementBed.xml b/tools/bedtools/complementBed.xml
@@ -1,4 +1,4 @@
-<tool id="bedtools_complementbed" name="ComplementBed" version="@WRAPPER_VERSION@.0">
+<tool id="bedtools_complementbed" name="bedtools ComplementBed" version="@WRAPPER_VERSION@">
     <description>Extract intervals not represented by an interval file</description>
     <macros>
         <import>macros.xml</import>

diff --git a/tools/bedtools/coverageBed.xml b/tools/bedtools/coverageBed.xml
@@ -1,4 +1,4 @@
-<tool id="bedtools_coveragebed" name="Compute both the depth and breadth of coverage" version="@WRAPPER_VERSION@.3">
+<tool id="bedtools_coveragebed" name="bedtools Compute both the depth and breadth of coverage" version="@WRAPPER_VERSION@">
     <description>of features in file B on the features in file A (bedtools coverage)</description>
     <macros>
         <import>macros.xml</import>

diff --git a/tools/bedtools/expandBed.xml b/tools/bedtools/expandBed.xml
@@ -1,4 +1,4 @@
-<tool id="bedtools_expandbed" name="ExpandBed" version="@WRAPPER_VERSION@.0">
+<tool id="bedtools_expandbed" name="bedtools ExpandBed" version="@WRAPPER_VERSION@">
     <description>replicate lines based on lists of values in columns</description>
     <macros>
         <import>macros.xml</import>

diff --git a/tools/bedtools/fisherBed.xml b/tools/bedtools/fisherBed.xml
@@ -1,4 +1,4 @@
-<tool id="bedtools_fisher" name="FisherBed" version="@WRAPPER_VERSION@.0">
+<tool id="bedtools_fisher" name="bedtools FisherBed" version="@WRAPPER_VERSION@">
     <description>calculate Fisher statistic between two feature files</description>
     <macros>
         <import>macros.xml</import>

diff --git a/tools/bedtools/flankBed.xml b/tools/bedtools/flankBed.xml
@@ -1,4 +1,4 @@
-<tool id="bedtools_flankbed" name="FlankBed" version="@WRAPPER_VERSION@.0">
+<tool id="bedtools_flankbed" name="bedtools FlankBed" version="@WRAPPER_VERSION@">
     <description>create new intervals from the flanks of existing intervals</description>
     <macros>
         <import>macros.xml</import>

diff --git a/tools/bedtools/genomeCoverageBed.xml b/tools/bedtools/genomeCoverageBed.xml
@@ -1,4 +1,4 @@
-<tool id="bedtools_genomecoveragebed" name="Genome Coverage" version="@WRAPPER_VERSION@.0">
+<tool id="bedtools_genomecoveragebed" name="bedtools Genome Coverage" version="@WRAPPER_VERSION@">
     <description>compute the coverage over an entire genome</description>
     <macros>
         <import>macros.xml</import>

diff --git a/tools/bedtools/getfastaBed.xml b/tools/bedtools/getfastaBed.xml
@@ -1,4 +1,4 @@
-<tool id="bedtools_getfastabed" name="GetFastaBed" version="@WRAPPER_VERSION@.1">
+<tool id="bedtools_getfastabed" name="bedtools GetFastaBed" version="@WRAPPER_VERSION@">
     <description>use intervals to extract sequences from a FASTA file</description>
     <macros>
         <import>macros.xml</import>

diff --git a/tools/bedtools/groupbyBed.xml b/tools/bedtools/groupbyBed.xml
@@ -1,4 +1,4 @@
-<tool id="bedtools_groupbybed" name="GroupByBed" version="@WRAPPER_VERSION@.0">
+<tool id="bedtools_groupbybed" name="bedtools GroupByBed" version="@WRAPPER_VERSION@">
     <description>group by common cols and summarize other cols</description>
     <macros>
         <import>macros.xml</import>