From 8cf87fa507937e9762452110ec3e4fe011deb2eb Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 4 Nov 2024 17:07:58 -0500 Subject: [PATCH 1/5] Add `mem` for all tools with `cores` and rescale some memory based on usegalaxy.org usage --- tools.yml | 185 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 96 insertions(+), 89 deletions(-) diff --git a/tools.yml b/tools.yml index 3a44773..38d360c 100644 --- a/tools.yml +++ b/tools.yml @@ -2,6 +2,7 @@ global: default_inherits: default tools: default: + abstract: true gpus: 0 cores: 1 mem: cores * 3.8 @@ -30,15 +31,17 @@ tools: cores: 1 mem: 20 toolshed.g2.bx.psu.edu/repos/artbio/repenrich/repenrich/.*: + # noqa: T102 cores: 8 toolshed.g2.bx.psu.edu/repos/bebatut/cdhit/cd_hit_est/.*: + # noqa: T102 cores: 4 toolshed.g2.bx.psu.edu/repos/bgruening/alevin/alevin/.*: cores: 8 mem: 92 toolshed.g2.bx.psu.edu/repos/bgruening/antismash/antismash/.*: cores: 10 - mem: 90 + mem: 24 env: _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G toolshed.g2.bx.psu.edu/repos/bgruening/augustus/augustus/.*: @@ -108,7 +111,7 @@ tools: mem: 24 toolshed.g2.bx.psu.edu/repos/bgruening/deeptools_bam_coverage/deeptools_bam_coverage/.*: cores: 10 - mem: 24 + mem: 14 toolshed.g2.bx.psu.edu/repos/bgruening/deeptools_bam_pe_fragmentsize/deeptools_bam_pe_fragmentsize/.*: cores: 10 mem: 24 @@ -120,7 +123,7 @@ tools: mem: 24 toolshed.g2.bx.psu.edu/repos/bgruening/deeptools_compute_matrix/deeptools_compute_matrix/.*: cores: 10 - mem: 30 + mem: 24 toolshed.g2.bx.psu.edu/repos/bgruening/deeptools_correct_gc_bias/deeptools_correct_gc_bias/.*: cores: 10 mem: 24 @@ -240,6 +243,7 @@ tools: mem: 80 toolshed.g2.bx.psu.edu/repos/bgruening/keras_train_and_eval/keras_train_and_eval/.*: cores: 4 + mem: 7 env: CUDA_VISIBLE_DEVICES: 0 toolshed.g2.bx.psu.edu/repos/bgruening/minced/minced/.*: @@ -257,6 +261,7 @@ tools: cores: 5 mem: 12 toolshed.g2.bx.psu.edu/repos/bgruening/nanopolish_polya/nanopolish_polya/.*: + # noqa: T102 cores: 3 toolshed.g2.bx.psu.edu/repos/bgruening/nanopolish_variants/nanopolish_variants/.*: cores: 5 @@ -266,6 +271,7 @@ tools: toolshed.g2.bx.psu.edu/repos/bgruening/numeric_clustering/numeric_clustering/.*: mem: 12 toolshed.g2.bx.psu.edu/repos/bgruening/openduck_run_smd/openduck_run_smd/.*: + # noqa: T102 gpus: 1 cores: 1 env: @@ -298,10 +304,10 @@ tools: toolshed.g2.bx.psu.edu/repos/bgruening/rxdock_sort_filter/rxdock_sort_filter/.*: mem: 90 toolshed.g2.bx.psu.edu/repos/bgruening/sailfish/sailfish/.*: - cores: 6 - mem: 70 + cores: 8 + mem: 48 toolshed.g2.bx.psu.edu/repos/bgruening/salmon/salmon/.*: - cores: 6 + cores: 8 mem: 70 toolshed.g2.bx.psu.edu/repos/bgruening/sklearn_model_validation/sklearn_model_validation/.*: cores: 10 @@ -325,12 +331,15 @@ tools: mem: 24 toolshed.g2.bx.psu.edu/repos/bgruening/trim_galore/trim_galore/.*: cores: 5 + mem: 20 toolshed.g2.bx.psu.edu/repos/bgruening/xchem_pose_scoring/xchem_pose_scoring/.*: + # noqa: T102 gpus: 1 cores: 1 env: CUDA_VISIBLE_DEVICES: 0 toolshed.g2.bx.psu.edu/repos/bgruening/xchem_transfs_scoring/xchem_transfs_scoring/.*: + # noqa: T102 gpus: 1 cores: 1 env: @@ -356,6 +365,7 @@ tools: mem: 8 toolshed.g2.bx.psu.edu/repos/chemteam/gmx_sim/gmx_sim/.*: cores: 8 + mem: 8 toolshed.g2.bx.psu.edu/repos/chemteam/md_converter/md_converter/.*: mem: 16 toolshed.g2.bx.psu.edu/repos/chemteam/mdanalysis_cosine_analysis/mdanalysis_cosine_analysis/.*: @@ -390,7 +400,7 @@ tools: - docker toolshed.g2.bx.psu.edu/repos/crs4/prokka/prokka/.*: cores: 8 - mem: 20 + mem: 12 env: _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G scheduling: @@ -403,6 +413,7 @@ tools: mem: 12 toolshed.g2.bx.psu.edu/repos/csbl/repeatmodeler/repeatmodeler/.*: cores: 8 + mem: 14 toolshed.g2.bx.psu.edu/repos/devteam/bamtools_filter/bamFilter/.*: mem: 7.6 toolshed.g2.bx.psu.edu/repos/devteam/bowtie2/bowtie2/.*: @@ -418,7 +429,7 @@ tools: mem: 10 toolshed.g2.bx.psu.edu/repos/devteam/cuffdiff/cuffdiff/.*: cores: 6 - mem: 40 + mem: 16 toolshed.g2.bx.psu.edu/repos/devteam/cufflinks/cufflinks/.*: cores: 6 mem: 30 @@ -442,18 +453,19 @@ tools: toolshed.g2.bx.psu.edu/repos/devteam/fastq_groomer/fastq_groomer/.*: mem: 7.6 toolshed.g2.bx.psu.edu/repos/devteam/fastq_paired_end_deinterlacer/fastq_paired_end_deinterlacer/.*: - mem: 30.4 + mem: 24 toolshed.g2.bx.psu.edu/repos/devteam/fastq_paired_end_interlacer/fastq_paired_end_interlacer/.*: mem: 30.4 toolshed.g2.bx.psu.edu/repos/devteam/fastq_paired_end_joiner/fastq_paired_end_joiner/.*: mem: 11.4 toolshed.g2.bx.psu.edu/repos/devteam/fastqc/fastqc/.*: - cores: 8 + cores: min(max(int(input_size * 4), 1), 16) + mem: min(max(int(input_size * 7), 5.6), 64) toolshed.g2.bx.psu.edu/repos/devteam/fastx_collapser/cshl_fastx_collapser/.*: mem: 16 toolshed.g2.bx.psu.edu/repos/devteam/freebayes/freebayes/.*: cores: 10 - mem: 90 + mem: 48 toolshed.g2.bx.psu.edu/repos/devteam/gffread/gffread/.*: mem: 34.2 toolshed.g2.bx.psu.edu/repos/devteam/hisat/hisat/.*: @@ -472,6 +484,7 @@ tools: mem: 40 toolshed.g2.bx.psu.edu/repos/devteam/ncbi_blast_plus/ncbi_blastp_wrapper/.*: cores: 8 + mem: 40 rules: - id: pulsar_rule if: | @@ -492,82 +505,15 @@ tools: mem: 40 toolshed.g2.bx.psu.edu/repos/devteam/ncbi_blast_plus/ncbi_tblastx_wrapper/.*: cores: 16 + mem: 64 toolshed.g2.bx.psu.edu/repos/devteam/picard/.*: cores: 3 - env: - _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G - toolshed.g2.bx.psu.edu/repos/devteam/picard/PicardASMetrics/.*: - mem: 12 - env: - _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G - toolshed.g2.bx.psu.edu/repos/devteam/picard/PicardGCBiasMetrics/.*: - mem: 12 - env: - _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G - toolshed.g2.bx.psu.edu/repos/devteam/picard/PicardHsMetrics/.*: mem: 12 env: _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G - toolshed.g2.bx.psu.edu/repos/devteam/picard/PicardInsertSize/.*: - mem: 12 - env: - _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_ARRG/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_AddOrReplaceReadGroups/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_BamIndexStats/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_CASM/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_CleanSam/.*: - mem: 12 - env: - _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_CollectInsertSizeMetrics/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_CollectRnaSeqMetrics/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_DownsampleSam/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_EstimateLibraryComplexity/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_FastqToSam/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_FilterSamReads/.*: - mem: 12 - env: TMP_DIR: $TMPDIR - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_FixMateInformation/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_MarkDuplicates/.*: - mem: 12 - env: - _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_MergeSamFiles/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_NormalizeFasta/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_QualityScoreDistribution/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_ReorderSam/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_ReplaceSamHeader/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_SamToFastq/.*: - mem: 12 toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_SortSam/.*: mem: 10 - env: - _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G - toolshed.g2.bx.psu.edu/repos/devteam/picard/rgEstLibComp/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/devteam/picard/rgPicFixMate/.*: - mem: 12 - toolshed.g2.bx.psu.edu/repos/devteam/picard/rgPicardMarkDups/.*: - mem: 12 - env: - _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G toolshed.g2.bx.psu.edu/repos/devteam/sam_merge/sam_merge2/.*: mem: 32 env: @@ -579,6 +525,7 @@ tools: mem: 7.6 toolshed.g2.bx.psu.edu/repos/devteam/samtools_mpileup/samtools_mpileup/.*: cores: 5 + mem: 24 toolshed.g2.bx.psu.edu/repos/devteam/samtools_rmdup/samtools_rmdup/.*: mem: 7.6 toolshed.g2.bx.psu.edu/repos/devteam/samtools_sort/samtools_sort/.*: @@ -588,8 +535,8 @@ tools: cores: 2 mem: 7.6 toolshed.g2.bx.psu.edu/repos/devteam/tophat2/tophat2/.*: - cores: 10 - mem: 90 + cores: 16 + mem: 64 toolshed.g2.bx.psu.edu/repos/devteam/velvet/velvetg/.*: mem: 32 env: @@ -639,10 +586,12 @@ tools: cores: 24 mem: 200 toolshed.g2.bx.psu.edu/repos/galaxy-australia/dorado_pod5_convert/dorado_pod5_convert/.*: + # noqa: T102 cores: 8 toolshed.g2.bx.psu.edu/repos/galaxy-australia/hapcut2/hapcut2/.*: mem: 8 toolshed.g2.bx.psu.edu/repos/galaxy-australia/hifiadapterfilt/hifiadapterfilt/.*: + # noqa: T102 cores: 12 toolshed.g2.bx.psu.edu/repos/galaxy-australia/hifiasm_meta/hifiasm_meta/.*: cores: 16 @@ -699,10 +648,13 @@ tools: mem: 92 toolshed.g2.bx.psu.edu/repos/galaxyp/maxquant/maxquant/.*: cores: 16 + mem: 64 toolshed.g2.bx.psu.edu/repos/galaxyp/maxquant/maxquant_mqpar/.*: cores: 16 + mem: 64 toolshed.g2.bx.psu.edu/repos/galaxyp/maxquant_mqpar/maxquant_mqpar/.*: cores: 16 + mem: 64 toolshed.g2.bx.psu.edu/repos/galaxyp/metagene_annotator/metagene_annotator/.*: mem: 16 toolshed.g2.bx.psu.edu/repos/galaxyp/morpheus/morpheus/.*: @@ -1107,13 +1059,16 @@ tools: toolshed.g2.bx.psu.edu/repos/galaxyp/uniprotxml_downloader/uniprotxml_downloader/.*: mem: 8 toolshed.g2.bx.psu.edu/repos/gbcs-embl-heidelberg/je_clip/je_clip/.*: + # noqa: T102 cores: 8 toolshed.g2.bx.psu.edu/repos/gbcs-embl-heidelberg/je_demultiplex/je_demultiplex/.*: cores: 8 mem: 20 toolshed.g2.bx.psu.edu/repos/gbcs-embl-heidelberg/je_demultiplex_illu/je_demultiplex_illu/.*: + # noqa: T102 cores: 8 toolshed.g2.bx.psu.edu/repos/gbcs-embl-heidelberg/je_markdupes/je_markdupes/.*: + # noqa: T102 cores: 8 toolshed.g2.bx.psu.edu/repos/genouest/braker3/braker3/.*: cores: 16 @@ -1140,12 +1095,14 @@ tools: mem: 70 toolshed.g2.bx.psu.edu/repos/iuc/anndata_export/anndata_export/.*: cores: 4 + mem: 8 toolshed.g2.bx.psu.edu/repos/iuc/anndata_import/anndata_import/.*: mem: 2 + input_size * 10 toolshed.g2.bx.psu.edu/repos/iuc/anndata_inspect/anndata_inspect/.*: mem: 15.2 toolshed.g2.bx.psu.edu/repos/iuc/anndata_manipulate/anndata_manipulate/.*: cores: 4 + mem: 12 toolshed.g2.bx.psu.edu/repos/iuc/annotatemyids/annotatemyids/.*: mem: 8 toolshed.g2.bx.psu.edu/repos/iuc/arriba/arriba/.*: @@ -1159,6 +1116,7 @@ tools: toolshed.g2.bx.psu.edu/repos/iuc/bandage/bandage_info/.*: mem: 16 toolshed.g2.bx.psu.edu/repos/iuc/barrnap/barrnap/.*: + # noqa: T102 cores: 3 toolshed.g2.bx.psu.edu/repos/iuc/bbtools_callvariants/bbtools_callvariants/.*: mem: 15 @@ -1166,8 +1124,10 @@ tools: _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G toolshed.g2.bx.psu.edu/repos/iuc/bcftools_mpileup/bcftools_mpileup/.*: cores: 7 + mem: 20 toolshed.g2.bx.psu.edu/repos/iuc/bcftools_norm/bcftools_norm/.*: cores: 2 + mem: 8 toolshed.g2.bx.psu.edu/repos/iuc/bedtools/bedtools_intersectbed/.*: mem: 40 toolshed.g2.bx.psu.edu/repos/iuc/bedtools/bedtools_mergebed/.*: @@ -1208,10 +1168,10 @@ tools: mem: 100 toolshed.g2.bx.psu.edu/repos/iuc/chira_map/chira_map/.*: cores: 10 - mem: 80 + mem: 24 toolshed.g2.bx.psu.edu/repos/iuc/chira_merge/chira_merge/.*: cores: 1 - mem: 60 + mem: 12 toolshed.g2.bx.psu.edu/repos/iuc/chira_quantify/chira_quantify/.*: cores: 1 mem: 60 @@ -1250,6 +1210,7 @@ tools: mem: 8 toolshed.g2.bx.psu.edu/repos/iuc/dexseq/dexseq/.*: cores: 8 + mem: 32 toolshed.g2.bx.psu.edu/repos/iuc/dexseq/dexseq_count/.*: mem: 25 toolshed.g2.bx.psu.edu/repos/iuc/disco/disco/.*: @@ -1269,6 +1230,7 @@ tools: mem: 8 toolshed.g2.bx.psu.edu/repos/iuc/exonerate/exonerate/.*: cores: 30 + mem: 24 toolshed.g2.bx.psu.edu/repos/iuc/fasta_stats/fasta-stats/.*: mem: 11.4 toolshed.g2.bx.psu.edu/repos/iuc/fastp/fastp/.*: @@ -1355,6 +1317,7 @@ tools: mem: 30.4 toolshed.g2.bx.psu.edu/repos/iuc/gemini_inheritance/gemini_inheritance/.*: cores: 4 + mem: 8 env: MKL_NUM_THREADS: '{cores}' NUMBA_NUM_THREADS: '{cores}' @@ -1372,6 +1335,7 @@ tools: mem: 122.8 toolshed.g2.bx.psu.edu/repos/iuc/gubbins/gubbins/.*: cores: 16 + mem: 12 toolshed.g2.bx.psu.edu/repos/iuc/hisat2/hisat2/.*: cores: 8 mem: 20 @@ -1412,6 +1376,7 @@ tools: mem: 10 toolshed.g2.bx.psu.edu/repos/iuc/hyphy_prime/hyphy_prime/.*: cores: 20 + mem: 10 toolshed.g2.bx.psu.edu/repos/iuc/hyphy_relax/hyphy_relax/.*: cores: 20 mem: 10 @@ -1428,6 +1393,7 @@ tools: mem: 24 toolshed.g2.bx.psu.edu/repos/iuc/iqtree/iqtree/.*: cores: 10 + mem: 7 toolshed.g2.bx.psu.edu/repos/iuc/isoformswitchanalyzer/isoformswitchanalyzer/.*: mem: 10 toolshed.g2.bx.psu.edu/repos/iuc/ivar_consensus/ivar_consensus/.*: @@ -1450,6 +1416,7 @@ tools: mem: 20 toolshed.g2.bx.psu.edu/repos/iuc/kc_align/kc-align/.*: cores: 3 + mem: 12 toolshed.g2.bx.psu.edu/repos/iuc/khmer_abundance_distribution_single/khmer_abundance_distribution_single/.*: cores: 8 mem: 8 @@ -1474,11 +1441,13 @@ tools: mem: 11.4 toolshed.g2.bx.psu.edu/repos/iuc/lofreq_viterbi/lofreq_viterbi/.*: cores: 4 + mem: 16 toolshed.g2.bx.psu.edu/repos/iuc/lumpy_prep/lumpy_prep/.*: mem: 8 toolshed.g2.bx.psu.edu/repos/iuc/lumpy_sv/lumpy_sv/.*: mem: 8 toolshed.g2.bx.psu.edu/repos/iuc/maaslin2/maaslin2/1.16.0.*: + # noqa: T102 cores: 1 toolshed.g2.bx.psu.edu/repos/iuc/macs2/macs2_bdgcmp/.*: mem: 12 @@ -1526,6 +1495,7 @@ tools: mem: 48 toolshed.g2.bx.psu.edu/repos/iuc/metaphlan/metaphlan/.*: cores: 16 + mem: 58 toolshed.g2.bx.psu.edu/repos/iuc/metaphlan2/metaphlan2/.*: cores: 2 mem: 7.6 @@ -1612,6 +1582,7 @@ tools: TERM: vt100 toolshed.g2.bx.psu.edu/repos/iuc/mothur_chimera_vsearch/mothur_chimera_vsearch/.*: cores: 7 + mem: 8 toolshed.g2.bx.psu.edu/repos/iuc/mothur_chop_seqs/mothur_chop_seqs/.*: cores: 2 mem: 20 @@ -2188,6 +2159,7 @@ tools: mem: 10 toolshed.g2.bx.psu.edu/repos/iuc/mummer_nucmer/mummer_nucmer/.*: cores: 5 + mem: 12 toolshed.g2.bx.psu.edu/repos/iuc/nanocompore_sampcomp/nanocompore_sampcomp/.*: cores: 9 mem: 48 @@ -2204,6 +2176,7 @@ tools: if: input_size < 0.2 mem: 8 toolshed.g2.bx.psu.edu/repos/iuc/ont_fast5_api_compress_fast5/ont_fast5_api_compress_fast5/.*: + # noqa: T102 cores: 8 toolshed.g2.bx.psu.edu/repos/iuc/orfipy/orfipy/.*: cores: 8 @@ -2213,8 +2186,10 @@ tools: mem: 30.7 toolshed.g2.bx.psu.edu/repos/iuc/pangolin/pangolin/.*: cores: 8 + mem: 8 toolshed.g2.bx.psu.edu/repos/iuc/pear/iuc_pear/.*: cores: 7 + mem: 58 toolshed.g2.bx.psu.edu/repos/iuc/pharokka/pharokka/.*: cores: 8 mem: 31 @@ -2236,7 +2211,7 @@ tools: toolshed.g2.bx.psu.edu/repos/iuc/polypolish/polypolish/.*: mem: 12 toolshed.g2.bx.psu.edu/repos/iuc/porechop/porechop/.*: - mem: 40 + mem: 20 toolshed.g2.bx.psu.edu/repos/iuc/poretools_events/poretools_events/.*: mem: 11.4 toolshed.g2.bx.psu.edu/repos/iuc/poretools_extract/poretools_extract/.*: @@ -2269,6 +2244,7 @@ tools: toolshed.g2.bx.psu.edu/repos/iuc/purge_dups/purge_dups/.*: mem: 30.4 toolshed.g2.bx.psu.edu/repos/iuc/qualimap_bamqc/qualimap_bamqc/.*: + cores: 6 mem: 24 toolshed.g2.bx.psu.edu/repos/iuc/qualimap_rnaseq/qualimap_rnaseq/.*: mem: 12 @@ -2276,6 +2252,7 @@ tools: cores: 10 mem: 12 toolshed.g2.bx.psu.edu/repos/iuc/raven/raven/.*: + # noqa: T102 cores: 16 toolshed.g2.bx.psu.edu/repos/iuc/raxml/raxml/.*: cores: 16 @@ -2293,6 +2270,7 @@ tools: mem: 250 toolshed.g2.bx.psu.edu/repos/iuc/roary/roary/.*: cores: 24 + mem: 24 toolshed.g2.bx.psu.edu/repos/iuc/salsa/salsa/.*: cores: 1 mem: 12 @@ -2309,7 +2287,8 @@ tools: cores: 4 mem: 7.6 toolshed.g2.bx.psu.edu/repos/iuc/scanpy_remove_confounders/scanpy_remove_confounders/.*: - cores: 9 + cores: 8 + mem: 28 toolshed.g2.bx.psu.edu/repos/iuc/schicexplorer_schicadjustmatrix/schicexplorer_schicadjustmatrix/.*: cores: 5 mem: 64 @@ -2360,18 +2339,20 @@ tools: mem: 64 toolshed.g2.bx.psu.edu/repos/iuc/seqtk/seqtk_seq/.*: cores: 4 + mem: 12 toolshed.g2.bx.psu.edu/repos/iuc/seurat/seurat/.*: mem: 10 toolshed.g2.bx.psu.edu/repos/iuc/shasta/shasta/.*: cores: 10 mem: 12 toolshed.g2.bx.psu.edu/repos/iuc/shovill/shovill/.*: - cores: 4 - mem: 50 + cores: 8 + mem: 32 env: SHOVILL_RAM: '{int(mem)}' toolshed.g2.bx.psu.edu/repos/iuc/slamdunk/slamdunk/.*: cores: 10 + mem: 30 toolshed.g2.bx.psu.edu/repos/iuc/snippy/snippy/.*: mem: 12 toolshed.g2.bx.psu.edu/repos/iuc/snippy/snippy_core/.*: @@ -2421,33 +2402,41 @@ tools: mem: 4 toolshed.g2.bx.psu.edu/repos/iuc/sra_tools/fasterq_dump/.*: cores: 8 + mem: 28 toolshed.g2.bx.psu.edu/repos/iuc/sra_tools/fastq_dump/.*: mem: 20 toolshed.g2.bx.psu.edu/repos/iuc/sra_tools/sam_dump/.*: - cores: 3 + cores: 4 + mem: 8 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_clonefilter/stacks2_clonefilter/.*: mem: 8 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_cstacks/stacks2_cstacks/.*: + # noqa: T102 cores: 16 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_denovomap/stacks2_denovomap/.*: cores: 4 mem: 16 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_gstacks/stacks2_gstacks/.*: + # noqa: T102 cores: 16 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_kmerfilter/stacks2_kmerfilter/.*: mem: 7.6 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_populations/stacks2_populations/.*: + # noqa: T102 cores: 8 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_refmap/stacks2_refmap/.*: + # noqa: T102 cores: 8 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_shortreads/stacks2_shortreads/.*: mem: 7.6 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_sstacks/stacks2_sstacks/.*: + # noqa: T102 cores: 16 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_tsv2bam/stacks2_tsv2bam/.*: cores: 8 mem: 42 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_ustacks/stacks2_ustacks/.*: + # noqa: T102 cores: 16 toolshed.g2.bx.psu.edu/repos/iuc/stacks_clonefilter/stacks_clonefilter/.*: mem: 8 @@ -2478,17 +2467,23 @@ tools: mem: 8 toolshed.g2.bx.psu.edu/repos/iuc/trinity_align_and_estimate_abundance/trinity_align_and_estimate_abundance/.*: cores: 8 + mem: 38 toolshed.g2.bx.psu.edu/repos/iuc/trinity_run_de_analysis/trinity_run_de_analysis/.*: mem: 15 toolshed.g2.bx.psu.edu/repos/iuc/trycycler_cluster/trycycler_cluster/.*: + # noqa: T102 cores: 9 toolshed.g2.bx.psu.edu/repos/iuc/trycycler_consensus/trycycler_consensus/.*: + # noqa: T102 cores: 16 toolshed.g2.bx.psu.edu/repos/iuc/trycycler_partition/trycycler_partition/.*: + # noqa: T102 cores: 9 toolshed.g2.bx.psu.edu/repos/iuc/trycycler_reconcile_msa/trycycler_reconcile_msa/.*: + # noqa: T102 cores: 9 toolshed.g2.bx.psu.edu/repos/iuc/trycycler_subsample/trycycler_subsample/.*: + # noqa: T102 cores: 9 toolshed.g2.bx.psu.edu/repos/iuc/ucsc_wigtobigwig/ucsc_wigtobigwig/.*: mem: 20 @@ -2500,7 +2495,7 @@ tools: mem: 12 toolshed.g2.bx.psu.edu/repos/iuc/unicycler/unicycler/.*: cores: 16 - mem: 90 + mem: 60 env: TERM: vt100 _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G @@ -2613,6 +2608,7 @@ tools: cores: 4 mem: 32 toolshed.g2.bx.psu.edu/repos/recetox/recetox_aplcms_align_features/recetox_aplcms_align_features/.*: + # noqa: T102 cores: 4 toolshed.g2.bx.psu.edu/repos/recetox/recetox_aplcms_generate_feature_table/recetox_aplcms_generate_feature_table/.*: mem: 16 @@ -2639,6 +2635,7 @@ tools: mem: 16 toolshed.g2.bx.psu.edu/repos/rnateam/mafft/rbc_mafft/.*: cores: 8 + mem: 12 toolshed.g2.bx.psu.edu/repos/rnateam/mafft/rbc_mafft_add/.*: mem: 12 toolshed.g2.bx.psu.edu/repos/rnateam/metilene/metilene/.*: @@ -2663,6 +2660,7 @@ tools: cores: 12 mem: 16 toolshed.g2.bx.psu.edu/repos/rnateam/rnalien/RNAlien/.*: + # noqa: T102 cores: 10 toolshed.g2.bx.psu.edu/repos/rnateam/segemehl/segemehl/.*: mem: 80 @@ -2675,6 +2673,7 @@ tools: mem: 26.6 toolshed.g2.bx.psu.edu/repos/simon-gladman/velvetoptimiser/velvetoptimiser/.*: cores: 4 + mem: 14 toolshed.g2.bx.psu.edu/repos/tyty/structurefold/iterative_map_pipeline/.*: mem: 60 toolshed.g2.bx.psu.edu/repos/wolma/mimodd_aln/mimodd_align/.*: @@ -2682,6 +2681,7 @@ tools: toolshed.g2.bx.psu.edu/repos/wolma/mimodd_main/mimodd_varcall/.*: mem: 22.8 .*bowtie2_index_builder_data_manager.*: + # noqa: T102 cores: 16 .*bwa_mem_index_builder_data_manager.*: cores: 12 @@ -2693,10 +2693,13 @@ tools: cores: 1 mem: 64 .*data_manager_cat.*: + # noqa: T102 cores: 5 .*data_manager_diamond_database_builder.*: + # noqa: T102 cores: 5 .*data_manager_eggnog_mapper_abspath.*: + # noqa: T102 cores: 5 .*data_manager_funannotate.*: cores: 1 @@ -2704,12 +2707,15 @@ tools: .*data_manager_gemini_download.*: mem: 20 .*data_manager_humann2_database_downloader.*: + # noqa: T102 cores: 5 .*data_manager_humann2_download.*: mem: 25 .*data_manager_humann_database_downloader.*: + # noqa: T102 cores: 5 .*data_manager_interproscan.*: + # noqa: T102 cores: 5 .*data_manager_metaphlan_download.*: cores: 12 @@ -2730,6 +2736,7 @@ tools: cores: 12 mem: 92 .*kraken2_build_database.*: + # noqa: T102 cores: 16 .*kraken_database_builder.*: mem: 200 From fb30e09a9353c924308154662c1185d6125f5807 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 4 Nov 2024 18:00:19 -0500 Subject: [PATCH 2/5] Replace noqas with FIXMEs --- tools.yml | 57 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/tools.yml b/tools.yml index 38d360c..05134f2 100644 --- a/tools.yml +++ b/tools.yml @@ -31,10 +31,10 @@ tools: cores: 1 mem: 20 toolshed.g2.bx.psu.edu/repos/artbio/repenrich/repenrich/.*: - # noqa: T102 + # FIXME: needs mem cores: 8 toolshed.g2.bx.psu.edu/repos/bebatut/cdhit/cd_hit_est/.*: - # noqa: T102 + # FIXME: needs mem cores: 4 toolshed.g2.bx.psu.edu/repos/bgruening/alevin/alevin/.*: cores: 8 @@ -261,7 +261,7 @@ tools: cores: 5 mem: 12 toolshed.g2.bx.psu.edu/repos/bgruening/nanopolish_polya/nanopolish_polya/.*: - # noqa: T102 + # FIXME: needs mem cores: 3 toolshed.g2.bx.psu.edu/repos/bgruening/nanopolish_variants/nanopolish_variants/.*: cores: 5 @@ -586,12 +586,12 @@ tools: cores: 24 mem: 200 toolshed.g2.bx.psu.edu/repos/galaxy-australia/dorado_pod5_convert/dorado_pod5_convert/.*: - # noqa: T102 + # FIXME: needs mem cores: 8 toolshed.g2.bx.psu.edu/repos/galaxy-australia/hapcut2/hapcut2/.*: mem: 8 toolshed.g2.bx.psu.edu/repos/galaxy-australia/hifiadapterfilt/hifiadapterfilt/.*: - # noqa: T102 + # FIXME: needs mem cores: 12 toolshed.g2.bx.psu.edu/repos/galaxy-australia/hifiasm_meta/hifiasm_meta/.*: cores: 16 @@ -1059,16 +1059,16 @@ tools: toolshed.g2.bx.psu.edu/repos/galaxyp/uniprotxml_downloader/uniprotxml_downloader/.*: mem: 8 toolshed.g2.bx.psu.edu/repos/gbcs-embl-heidelberg/je_clip/je_clip/.*: - # noqa: T102 + # FIXME: needs mem cores: 8 toolshed.g2.bx.psu.edu/repos/gbcs-embl-heidelberg/je_demultiplex/je_demultiplex/.*: cores: 8 mem: 20 toolshed.g2.bx.psu.edu/repos/gbcs-embl-heidelberg/je_demultiplex_illu/je_demultiplex_illu/.*: - # noqa: T102 + # FIXME: needs mem cores: 8 toolshed.g2.bx.psu.edu/repos/gbcs-embl-heidelberg/je_markdupes/je_markdupes/.*: - # noqa: T102 + # FIXME: needs mem cores: 8 toolshed.g2.bx.psu.edu/repos/genouest/braker3/braker3/.*: cores: 16 @@ -1088,6 +1088,9 @@ tools: mem: 12 env: _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G + toolshed.g2.bx.psu.edu/repos/iuc/extract_genomic_dna/Extract genomic DNA 1: + cores: 3 + mem: 3.7 toolshed.g2.bx.psu.edu/repos/iuc/abricate/abricate/.*: mem: 30.4 toolshed.g2.bx.psu.edu/repos/iuc/abyss/abyss-pe/.*: @@ -1116,7 +1119,7 @@ tools: toolshed.g2.bx.psu.edu/repos/iuc/bandage/bandage_info/.*: mem: 16 toolshed.g2.bx.psu.edu/repos/iuc/barrnap/barrnap/.*: - # noqa: T102 + # FIXME: needs mem cores: 3 toolshed.g2.bx.psu.edu/repos/iuc/bbtools_callvariants/bbtools_callvariants/.*: mem: 15 @@ -1235,6 +1238,7 @@ tools: mem: 11.4 toolshed.g2.bx.psu.edu/repos/iuc/fastp/fastp/.*: cores: 4 + mem: toolshed.g2.bx.psu.edu/repos/iuc/fasttree/fasttree/.*: mem: 30 rules: @@ -2176,7 +2180,7 @@ tools: if: input_size < 0.2 mem: 8 toolshed.g2.bx.psu.edu/repos/iuc/ont_fast5_api_compress_fast5/ont_fast5_api_compress_fast5/.*: - # noqa: T102 + # FIXME: needs mem cores: 8 toolshed.g2.bx.psu.edu/repos/iuc/orfipy/orfipy/.*: cores: 8 @@ -2252,7 +2256,7 @@ tools: cores: 10 mem: 12 toolshed.g2.bx.psu.edu/repos/iuc/raven/raven/.*: - # noqa: T102 + # FIXME: needs mem cores: 16 toolshed.g2.bx.psu.edu/repos/iuc/raxml/raxml/.*: cores: 16 @@ -2411,32 +2415,32 @@ tools: toolshed.g2.bx.psu.edu/repos/iuc/stacks2_clonefilter/stacks2_clonefilter/.*: mem: 8 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_cstacks/stacks2_cstacks/.*: - # noqa: T102 + # FIXME: needs mem cores: 16 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_denovomap/stacks2_denovomap/.*: cores: 4 mem: 16 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_gstacks/stacks2_gstacks/.*: - # noqa: T102 + # FIXME: needs mem cores: 16 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_kmerfilter/stacks2_kmerfilter/.*: mem: 7.6 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_populations/stacks2_populations/.*: - # noqa: T102 + # FIXME: needs mem cores: 8 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_refmap/stacks2_refmap/.*: - # noqa: T102 + # FIXME: needs mem cores: 8 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_shortreads/stacks2_shortreads/.*: mem: 7.6 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_sstacks/stacks2_sstacks/.*: - # noqa: T102 + # FIXME: needs mem cores: 16 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_tsv2bam/stacks2_tsv2bam/.*: cores: 8 mem: 42 toolshed.g2.bx.psu.edu/repos/iuc/stacks2_ustacks/stacks2_ustacks/.*: - # noqa: T102 + # FIXME: needs mem cores: 16 toolshed.g2.bx.psu.edu/repos/iuc/stacks_clonefilter/stacks_clonefilter/.*: mem: 8 @@ -2471,19 +2475,19 @@ tools: toolshed.g2.bx.psu.edu/repos/iuc/trinity_run_de_analysis/trinity_run_de_analysis/.*: mem: 15 toolshed.g2.bx.psu.edu/repos/iuc/trycycler_cluster/trycycler_cluster/.*: - # noqa: T102 + # FIXME: needs mem cores: 9 toolshed.g2.bx.psu.edu/repos/iuc/trycycler_consensus/trycycler_consensus/.*: - # noqa: T102 + # FIXME: needs mem cores: 16 toolshed.g2.bx.psu.edu/repos/iuc/trycycler_partition/trycycler_partition/.*: - # noqa: T102 + # FIXME: needs mem cores: 9 toolshed.g2.bx.psu.edu/repos/iuc/trycycler_reconcile_msa/trycycler_reconcile_msa/.*: - # noqa: T102 + # FIXME: needs mem cores: 9 toolshed.g2.bx.psu.edu/repos/iuc/trycycler_subsample/trycycler_subsample/.*: - # noqa: T102 + # FIXME: needs mem cores: 9 toolshed.g2.bx.psu.edu/repos/iuc/ucsc_wigtobigwig/ucsc_wigtobigwig/.*: mem: 20 @@ -2542,7 +2546,8 @@ tools: toolshed.g2.bx.psu.edu/repos/lecorguille/xcms_retcor/abims_xcms_retcor/.*: mem: 32 toolshed.g2.bx.psu.edu/repos/lparsons/cutadapt/cutadapt/.*: - cores: 5 + cores: 8 + mem: 28 toolshed.g2.bx.psu.edu/repos/lparsons/htseq_count/htseq_count/.*: mem: 32 toolshed.g2.bx.psu.edu/repos/mbernt/maxbin2/maxbin2/.*: @@ -2608,7 +2613,7 @@ tools: cores: 4 mem: 32 toolshed.g2.bx.psu.edu/repos/recetox/recetox_aplcms_align_features/recetox_aplcms_align_features/.*: - # noqa: T102 + # FIXME: needs mem cores: 4 toolshed.g2.bx.psu.edu/repos/recetox/recetox_aplcms_generate_feature_table/recetox_aplcms_generate_feature_table/.*: mem: 16 @@ -2660,7 +2665,7 @@ tools: cores: 12 mem: 16 toolshed.g2.bx.psu.edu/repos/rnateam/rnalien/RNAlien/.*: - # noqa: T102 + # FIXME: needs mem cores: 10 toolshed.g2.bx.psu.edu/repos/rnateam/segemehl/segemehl/.*: mem: 80 @@ -2756,8 +2761,6 @@ tools: mem: 36 CONVERTER_bedgraph_to_bigwig: mem: 8 - Extract genomic DNA 1: - cores: 3 bfast_wrapper: cores: 10 mem: 20 From e04983cbfee8c313efe2841f74b7d86220140499 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 5 Nov 2024 12:03:44 -0500 Subject: [PATCH 3/5] Improvements based on review from @mira-miracoli --- tools.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools.yml b/tools.yml index 05134f2..d237fcb 100644 --- a/tools.yml +++ b/tools.yml @@ -464,8 +464,9 @@ tools: toolshed.g2.bx.psu.edu/repos/devteam/fastx_collapser/cshl_fastx_collapser/.*: mem: 16 toolshed.g2.bx.psu.edu/repos/devteam/freebayes/freebayes/.*: + # see https://github.com/usegalaxy-eu/infrastructure-playbook/pull/881 for some numbers cores: 10 - mem: 48 + mem: 9 + input_size * 1 toolshed.g2.bx.psu.edu/repos/devteam/gffread/gffread/.*: mem: 34.2 toolshed.g2.bx.psu.edu/repos/devteam/hisat/hisat/.*: @@ -1172,9 +1173,6 @@ tools: toolshed.g2.bx.psu.edu/repos/iuc/chira_map/chira_map/.*: cores: 10 mem: 24 - toolshed.g2.bx.psu.edu/repos/iuc/chira_merge/chira_merge/.*: - cores: 1 - mem: 12 toolshed.g2.bx.psu.edu/repos/iuc/chira_quantify/chira_quantify/.*: cores: 1 mem: 60 @@ -1238,7 +1236,7 @@ tools: mem: 11.4 toolshed.g2.bx.psu.edu/repos/iuc/fastp/fastp/.*: cores: 4 - mem: + mem: min(max(int(input_size * 6), 3.7), 58) toolshed.g2.bx.psu.edu/repos/iuc/fasttree/fasttree/.*: mem: 30 rules: @@ -1404,6 +1402,7 @@ tools: mem: 30.4 toolshed.g2.bx.psu.edu/repos/iuc/ivar_trim/ivar_trim/.*: cores: 8 + mem: 12 scheduling: accept: - pulsar @@ -2193,7 +2192,7 @@ tools: mem: 8 toolshed.g2.bx.psu.edu/repos/iuc/pear/iuc_pear/.*: cores: 7 - mem: 58 + mem: min(max(int(input_size * 8), 3.7), 28) toolshed.g2.bx.psu.edu/repos/iuc/pharokka/pharokka/.*: cores: 8 mem: 31 @@ -2350,8 +2349,9 @@ tools: cores: 10 mem: 12 toolshed.g2.bx.psu.edu/repos/iuc/shovill/shovill/.*: - cores: 8 - mem: 32 + # this should mirror spades + cores: 20 + mem: min(max(int(input_size * 32), 14), 240) env: SHOVILL_RAM: '{int(mem)}' toolshed.g2.bx.psu.edu/repos/iuc/slamdunk/slamdunk/.*: @@ -2575,7 +2575,7 @@ tools: mem: 8 toolshed.g2.bx.psu.edu/repos/nml/spades/spades/.*: cores: 20 - mem: 330 + mem: min(max(int(input_size * 32), 14), 240) toolshed.g2.bx.psu.edu/repos/pavlo-lutsik/rnbeads/rnbeads/.*: mem: 20 toolshed.g2.bx.psu.edu/repos/peterjc/blast2go/blast2go/.*: From 51b6fdf8e3acb7c30e5043d99bc11cb89860ddca Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Fri, 8 Nov 2024 13:14:51 -0500 Subject: [PATCH 4/5] Increase iqtree mem from review by @cat-bro --- tools.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.yml b/tools.yml index d237fcb..0afa9a9 100644 --- a/tools.yml +++ b/tools.yml @@ -1395,7 +1395,7 @@ tools: mem: 24 toolshed.g2.bx.psu.edu/repos/iuc/iqtree/iqtree/.*: cores: 10 - mem: 7 + mem: 19 toolshed.g2.bx.psu.edu/repos/iuc/isoformswitchanalyzer/isoformswitchanalyzer/.*: mem: 10 toolshed.g2.bx.psu.edu/repos/iuc/ivar_consensus/ivar_consensus/.*: From 0cfae499d24b98e914a1fdcbe04dd3db42049aeb Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 20 Nov 2024 14:00:26 -0500 Subject: [PATCH 5/5] Decrease tophat resources --- tools.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools.yml b/tools.yml index 0afa9a9..473d054 100644 --- a/tools.yml +++ b/tools.yml @@ -536,8 +536,8 @@ tools: cores: 2 mem: 7.6 toolshed.g2.bx.psu.edu/repos/devteam/tophat2/tophat2/.*: - cores: 16 - mem: 64 + cores: 8 + mem: 32 toolshed.g2.bx.psu.edu/repos/devteam/velvet/velvetg/.*: mem: 32 env: