From 1dba944b5f6b9dd82ad63f990c1a6882cc5280d4 Mon Sep 17 00:00:00 2001 From: Shane Giles Date: Fri, 19 Apr 2024 14:15:15 -0600 Subject: [PATCH 1/3] feat(IPVC-2344): add tabix to be installed in Dockerfile, use environment file to set SeqRepo version, remove cp from uta-extract --- Dockerfile | 2 +- docker-compose.yml | 6 +++--- sbin/uta-extract | 5 ----- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index e58df99..4b83e36 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ FROM ubuntu:22.04 as uta ARG python_version="3.10" # list and install dependencies -ARG dependencies="python${python_version} python3-dev python3-pip rsync git postgresql-client-14" +ARG dependencies="python${python_version} python3-dev python3-pip rsync git postgresql-client-14 tabix" RUN apt-get update && apt-get install -y $dependencies && apt-get clean diff --git a/docker-compose.yml b/docker-compose.yml index b689645..b1a9e06 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,10 +23,10 @@ services: network_mode: host seqrepo-load: image: uta-update - command: sbin/seqrepo-load /usr/local/share/seqrepo 2024-02-20 /seqrepo-load/work /seqrepo-load/logs + command: sbin/seqrepo-load /usr/local/share/seqrepo ${UTA_ETL_OLD_SEQREPO_VERSION} /seqrepo-load/input /seqrepo-load/logs volumes: - ${UTA_ETL_SEQREPO_DIR}:/usr/local/share/seqrepo - - ${UTA_ETL_WORK_DIR}:/seqrepo-load/work + - ${UTA_ETL_NCBI_DIR}:/seqrepo-load/input - ${UTA_ETL_LOG_DIR}:/seqrepo-load/logs working_dir: /opt/repos/uta network_mode: host @@ -62,7 +62,7 @@ services: network_mode: host splign-manual: image: uta-update - command: sbin/uta-splign-manual 2024-02-20 ${UTA_ETL_OLD_UTA_VERSION} /uta-splign-manual/input /uta-splign-manual/work /uta-splign-manual/logs + command: sbin/uta-splign-manual ${UTA_ETL_OLD_SEQREPO_VERSION} ${UTA_ETL_OLD_UTA_VERSION} /uta-splign-manual/input /uta-splign-manual/work /uta-splign-manual/logs depends_on: uta: condition: service_healthy diff --git a/sbin/uta-extract b/sbin/uta-extract index a8d3d60..6a9ec24 100755 --- a/sbin/uta-extract +++ b/sbin/uta-extract @@ -36,8 +36,3 @@ sbin/ncbi_parse_genomic_gff.py "$GFF_files" | gzip -c > "$working_dir/unfiltered sbin/filter_exonset_transcripts.py --tx-info "$working_dir/txinfo.gz" --exonsets "$working_dir/unfiltered_exonsets.gz" \ --missing-ids "$working_dir/filtered_tx_acs.txt" | gzip -c > "$working_dir/exonsets.gz" 2>&1 | \ tee "$log_dir/filter_exonset_transcripts.log" - -# move fasta files into same dir -cp -f $ncbi_dir/refseq/H_sapiens/mRNA_Prot/human.*.rna.fna.gz $working_dir/ -cp -f $ncbi_dir/refseq/H_sapiens/mRNA_Prot/human.*.protein.faa.gz $working_dir/ -cp -f $ncbi_dir/genomes/refseq/vertebrate_mammalian/Homo_sapiens/all_assembly_versions/GCF_000001405*/GCF_*_genomic.fna.gz $working_dir/ From 9cf17b6abeaa1794722c3fc7bc188da31bb6be9c Mon Sep 17 00:00:00 2001 From: Shane Giles Date: Fri, 19 Apr 2024 14:30:16 -0600 Subject: [PATCH 2/3] feat(IPVC-2344): update seqrepo-load script to find fasta files instead of have a predetermined location --- sbin/seqrepo-load | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sbin/seqrepo-load b/sbin/seqrepo-load index 93942cf..cd6ae87 100755 --- a/sbin/seqrepo-load +++ b/sbin/seqrepo-load @@ -13,9 +13,9 @@ then exit 1 fi +fasta_files=$(find /seqrepo-load/input -type f -name "*.f[an]a.gz" -printf "%p ") + ## Load SeqRepo with new sequences seqrepo --root-directory "$seqrepo_root" \ load -n NCBI --instance-name "$seqrepo_version" \ - $sequence_dir/*.fna.gz \ - $sequence_dir/*.faa.gz 2>& 1 | \ - tee "$log_dir/seqrepo-load.log" + $fasta_files 2>& 1 | tee "$log_dir/seqrepo-load.log" From 9120128ac5057ad8216b70592eb0626ea305f175 Mon Sep 17 00:00:00 2001 From: Shane Giles Date: Wed, 24 Apr 2024 12:11:22 -0600 Subject: [PATCH 3/3] feat(IPVC-2344): after discussion reverting back to cp fasta files to working directory --- docker-compose.yml | 4 ++-- sbin/seqrepo-load | 6 +++--- sbin/uta-extract | 3 +++ 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b1a9e06..e74e13f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,10 +23,10 @@ services: network_mode: host seqrepo-load: image: uta-update - command: sbin/seqrepo-load /usr/local/share/seqrepo ${UTA_ETL_OLD_SEQREPO_VERSION} /seqrepo-load/input /seqrepo-load/logs + command: sbin/seqrepo-load /usr/local/share/seqrepo ${UTA_ETL_OLD_SEQREPO_VERSION} /seqrepo-load/work /seqrepo-load/logs volumes: - ${UTA_ETL_SEQREPO_DIR}:/usr/local/share/seqrepo - - ${UTA_ETL_NCBI_DIR}:/seqrepo-load/input + - ${UTA_ETL_WORK_DIR}:/seqrepo-load/work - ${UTA_ETL_LOG_DIR}:/seqrepo-load/logs working_dir: /opt/repos/uta network_mode: host diff --git a/sbin/seqrepo-load b/sbin/seqrepo-load index cd6ae87..adadc04 100755 --- a/sbin/seqrepo-load +++ b/sbin/seqrepo-load @@ -13,9 +13,9 @@ then exit 1 fi -fasta_files=$(find /seqrepo-load/input -type f -name "*.f[an]a.gz" -printf "%p ") - ## Load SeqRepo with new sequences seqrepo --root-directory "$seqrepo_root" \ load -n NCBI --instance-name "$seqrepo_version" \ - $fasta_files 2>& 1 | tee "$log_dir/seqrepo-load.log" + "$sequence_dir"/*.fna.gz \ + "$sequence_dir"/*.faa.gz 2>& 1 | \ + tee "$log_dir/seqrepo-load.log" diff --git a/sbin/uta-extract b/sbin/uta-extract index 6a9ec24..c4e239d 100755 --- a/sbin/uta-extract +++ b/sbin/uta-extract @@ -36,3 +36,6 @@ sbin/ncbi_parse_genomic_gff.py "$GFF_files" | gzip -c > "$working_dir/unfiltered sbin/filter_exonset_transcripts.py --tx-info "$working_dir/txinfo.gz" --exonsets "$working_dir/unfiltered_exonsets.gz" \ --missing-ids "$working_dir/filtered_tx_acs.txt" | gzip -c > "$working_dir/exonsets.gz" 2>&1 | \ tee "$log_dir/filter_exonset_transcripts.log" + +# move fasta files into same dir +find "$ncbi_dir" -type f -name "*.f[an]a.gz" -print0 | xargs -i --null cp {} "$working_dir/"