Updates and BGC Feature

.examples/_config_example.yaml

@@ -24,6 +24,9 @@ projects:
 # Project 2 (PEP file)
   - name: .examples/_pep_example/project_config.yaml
 
+bgc_projects:
+  - name: config/lanthipeptide/project_config.yaml
+
 #### GLOBAL RULE CONFIGURATION ####
 # This section configures the rules to run globally.
 # Use project specific rule configurations if you want to run different rules for each projects.

.examples/lactobacillus_delbruecki/project_config.yaml

@@ -0,0 +1,30 @@
+name: Lactobacillus_delbrueckii
+
+pep_version: 2.1.0
+
+description: "Lactobacillus delbrueckii 27 01 2023"
+
+sample_table: samples.csv
+
+#### RULE CONFIGURATION ####
+# rules: set value to TRUE if you want to run the analysis or FALSE if you don't
+rules:
+  seqfu: TRUE
+  mash: TRUE
+  fastani: TRUE
+  checkm: FALSE
+  gtdbtk: FALSE
+  prokka-gbk: TRUE
+  antismash: TRUE
+  query-bigslice: TRUE
+  bigscape: TRUE
+  bigslice: TRUE
+  automlst-wrapper: TRUE
+  arts: TRUE
+  roary: TRUE
+  eggnog: TRUE
+  eggnog-roary: TRUE
+  deeptfactor: TRUE
+  deeptfactor-roary: TRUE
+  cblaster-genome: TRUE
+  cblaster-bgc: TRUE

.examples/lactobacillus_delbruecki/samples.csv

@@ -0,0 +1,5 @@
+genome_id,source,organism,genus,species,strain,closest_placement_reference,input_file
+GCA_000056065.1,ncbi,,,,,,
+GCA_000182835.1,ncbi,,,,,,
+GCA_000191165.1,ncbi,,,,,,
+GCA_000014405.1,ncbi,,,,,,

.examples/lanthipeptide/df_antismash_6.1.1_bgc.csv

@@ -0,0 +1,5 @@
+bgc_id,genome_id,region,accession,start_pos,end_pos,contig_edge,product,region_length,source,gbk_path
+CR954253.1.region001,GCA_000056065.1,1.1,CR954253.1,17407,39909,False,['lanthipeptide-class-iii'],22502,bgcflow,
+CR954253.1.region002,GCA_000056065.1,1.2,CR954253.1,1745672,1767868,False,['lanthipeptide-class-iv'],22196,bgcflow,
+CP000156.1.region001,GCA_000191165.1,1.1,CP000156.1,1767251,1789447,False,['lanthipeptide-class-iv'],22196,bgcflow,
+CP000412.1.region001,GCA_000014405.1,1.1,CP000412.1,17283,39785,False,['lanthipeptide-class-iii'],22502,bgcflow,

.examples/lanthipeptide/project_config.yaml

@@ -0,0 +1,11 @@
+name: lanthipeptide_lactobacillus
+pep_version: 2.1.0
+description: 'A selection of lanthipeptides from Lactobacillus delbrueckii'
+sample_table: df_antismash_6.1.1_bgc.csv
+
+rules:
+  bigslice: TRUE
+  bigscape: TRUE
+  query-bigslice: TRUE
+  clinker: TRUE
+  interproscan: TRUE

.github/workflows/push.yml

@@ -23,7 +23,7 @@ jobs:
     - name: Linting
       uses: snakemake/[email protected]
       with:
-        directory: .test
+        directory: .tests
         snakefile: workflow/Snakefile
         stagein: "conda config --set channel_priority strict"
         args: "--lint"
@@ -41,7 +41,25 @@ jobs:
     - name: Dry-run workflow
       uses: snakemake/[email protected]
       with:
-        directory: .test
+        directory: .tests
         snakefile: workflow/Snakefile
         stagein: "conda config --set channel_priority strict"
         args: "--use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache -n"
+
+  unit-test:
+    runs-on: ubuntu-latest
+    needs:
+      - dry-run
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.x
+      - run: pip install git+https://github.com/NBChub/bgcflow_wrapper.git
+      - run: pip install pytest-cov
+      - name: Build coverage file
+        run: pytest --cov=.tests/unit .tests/unit/ > pytest-coverage.txt
+      - name: Comment coverage
+        uses: coroo/[email protected]
+        with:
+          pytest-coverage: pytest-coverage.txt

.gitignore

@@ -22,3 +22,5 @@ workflow/bgcflow/build*
 .panoptes.db
 notebooks/
 *.ipynb_checkpoints/
+plugins/
+metabase.db*

.tests/unit/antismash_overview_gather/data/config/config.yaml

@@ -0,0 +1,62 @@
+# This file should contain everything to configure the workflow on a global scale.
+
+#### PROJECT INFORMATION ####
+# This section control your project configuration.
+# Each project are separated by "-".
+# A project can be defined as (1) a yaml object or (2) a Portable Encapsulated Project (PEP) file.
+# (1) To define project as a yaml object, it must contain the variable "name" and "samples".
+#   - name : name of your project
+#   - samples : a csv file containing a list of genome ids for analysis with multiple sources mentioned. Genome ids must be unique.
+#   - rules: a yaml file containing project rule configurations. This will override global rule configuration.
+#   - prokka-db (optional): list of the custom accessions to use as prokka reference database.
+#   - gtdb-tax (optional): output summary file of GTDB-tk with "user_genome" and "classification" as the two minimum columns
+# (2) To define project using PEP file, only variable "name" should be given that points to the location of the PEP yaml file.
+#   - name: path to PEP .yaml file. See project example_pep for details.
+
+projects:
+# Project 1 (yaml object)
+#  - name: example
+#    samples: config/_genome_project_example/samples.csv
+#    rules: config/_genome_project_example/project_config.yaml
+#    prokka-db: config/_genome_project_example/prokka-db.csv
+#    gtdb-tax: config/_genome_project_example/gtdbtk.bac120.summary.tsv
+
+# Project 2 (PEP file)
+#  - name: config/_pep_example/project_config.yaml
+  - name: config/lactobacillus_delbruecki/project_config.yaml
+#### GLOBAL RULE CONFIGURATION ####
+# This section configures the rules to run globally.
+# Use project specific rule configurations if you want to run different rules for each projects.
+# rules: set value to TRUE if you want to run the analysis or FALSE if you don't
+rules:
+  seqfu: FALSE
+  mash: FALSE
+  fastani: FALSE
+  checkm: FALSE
+  gtdbtk: FALSE
+  prokka-gbk: FALSE
+  antismash: TRUE
+  query-bigslice: FALSE
+  bigscape: FALSE
+  bigslice: FALSE
+  automlst-wrapper: FALSE
+  arts: FALSE
+  roary: FALSE
+  eggnog: FALSE
+  eggnog-roary: FALSE
+  deeptfactor: FALSE
+  deeptfactor-roary: FALSE
+  cblaster-genome: FALSE
+  cblaster-bgc: FALSE
+
+#### RESOURCES CONFIGURATION ####
+# resources : the location of the resources to run the rule.
+# The default location is at "resources/{resource_name}".
+resources_path:
+  antismash_db: resources/antismash_db
+  eggnog_db: resources/eggnog_db
+  BiG-SCAPE: resources/BiG-SCAPE
+  bigslice: resources/bigslice
+  checkm: resources/checkm
+  gtdbtk: resources/gtdbtk
+  #RNAmmer: resources/RNAmmer # If specified, will override Barnapp in Prokka

.tests/unit/antismash_overview_gather/data/config/lactobacillus_delbruecki/project_config.yaml

@@ -0,0 +1,30 @@
+name: Lactobacillus_delbrueckii
+
+pep_version: 2.1.0
+
+description: "Lactobacillus delbrueckii 27 01 2023"
+
+sample_table: samples.csv
+
+#### RULE CONFIGURATION ####
+# rules: set value to TRUE if you want to run the analysis or FALSE if you don't
+rules:
+  seqfu: TRUE
+  mash: TRUE
+  fastani: TRUE
+  checkm: FALSE
+  gtdbtk: FALSE
+  prokka-gbk: TRUE
+  antismash: TRUE
+  query-bigslice: TRUE
+  bigscape: TRUE
+  bigslice: TRUE
+  automlst-wrapper: TRUE
+  arts: TRUE
+  roary: TRUE
+  eggnog: TRUE
+  eggnog-roary: TRUE
+  deeptfactor: TRUE
+  deeptfactor-roary: TRUE
+  cblaster-genome: TRUE
+  cblaster-bgc: TRUE

.tests/unit/antismash_overview_gather/data/config/lactobacillus_delbruecki/samples.csv

@@ -0,0 +1,5 @@
+genome_id,source,organism,genus,species,strain,closest_placement_reference,input_file
+GCA_000056065.1,ncbi,,,,,,
+GCA_000182835.1,ncbi,,,,,,
+GCA_000191165.1,ncbi,,,,,,
+GCA_000014405.1,ncbi,,,,,,

.tests/unit/antismash_overview_gather/data/data/interim/antismash/6.1.1/GCA_000014405.1_bgc_overview.json

@@ -0,0 +1,14 @@
+{
+  "CP000412.1.region001": {
+    "genome_id": "GCA_000014405.1",
+    "region": "1.1",
+    "accession": "CP000412.1",
+    "start_pos": "17283",
+    "end_pos": "39785",
+    "contig_edge": "False",
+    "product": [
+      "lanthipeptide-class-iii"
+    ],
+    "region_length": 22502
+  }
+}

.tests/unit/antismash_overview_gather/data/data/interim/antismash/6.1.1/GCA_000056065.1_bgc_overview.json

@@ -0,0 +1,26 @@
+{
+  "CR954253.1.region001": {
+    "genome_id": "GCA_000056065.1",
+    "region": "1.1",
+    "accession": "CR954253.1",
+    "start_pos": "17407",
+    "end_pos": "39909",
+    "contig_edge": "False",
+    "product": [
+      "lanthipeptide-class-iii"
+    ],
+    "region_length": 22502
+  },
+  "CR954253.1.region002": {
+    "genome_id": "GCA_000056065.1",
+    "region": "1.2",
+    "accession": "CR954253.1",
+    "start_pos": "1745672",
+    "end_pos": "1767868",
+    "contig_edge": "False",
+    "product": [
+      "lanthipeptide-class-iv"
+    ],
+    "region_length": 22196
+  }
+}

.tests/unit/antismash_overview_gather/data/data/interim/antismash/6.1.1/GCA_000182835.1_bgc_overview.json

@@ -0,0 +1 @@
+{}

.tests/unit/antismash_overview_gather/data/data/interim/antismash/6.1.1/GCA_000191165.1_bgc_overview.json

@@ -0,0 +1,14 @@
+{
+  "CP000156.1.region001": {
+    "genome_id": "GCA_000191165.1",
+    "region": "1.1",
+    "accession": "CP000156.1",
+    "start_pos": "1767251",
+    "end_pos": "1789447",
+    "contig_edge": "False",
+    "product": [
+      "lanthipeptide-class-iv"
+    ],
+    "region_length": 22196
+  }
+}

.tests/unit/antismash_overview_gather/data/data/interim/bgcflow_utils/samples.csv

@@ -0,0 +1,5 @@
+genome_id,source,organism,genus,species,strain,closest_placement_reference,input_file,ref_annotation,classification_source,sample_paths,prokka-db,gtdb_paths,name
+GCA_000056065.1,ncbi,,,,,,,,ncbi,config/lactobacillus_delbruecki/samples.csv,,,Lactobacillus_delbrueckii
+GCA_000182835.1,ncbi,,,,,,,,ncbi,['config/lactobacillus_delbruecki/samples.csv'],[''],[''],['Lactobacillus_delbrueckii']
+GCA_000191165.1,ncbi,,,,,,,,ncbi,['config/lactobacillus_delbruecki/samples.csv'],[''],[''],['Lactobacillus_delbrueckii']
+GCA_000014405.1,ncbi,,,,,,,,ncbi,['config/lactobacillus_delbruecki/samples.csv'],[''],[''],['Lactobacillus_delbrueckii']

.tests/unit/antismash_overview_gather/data/data/interim/bgcs/Lactobacillus_delbrueckii/6.1.1/GCA_000014405.1/GCA_000014405.1-change_log.json

@@ -0,0 +1,16 @@
+{
+    "GCA_000014405.1": {
+        "CP000412.1.region001.gbk": {
+            "record_id": "CP000412.1",
+            "original_id": "CP000412.1",
+            "target_path": "data/interim/antismash/6.1.1/GCA_000014405.1/CP000412.1.region001.gbk",
+            "symlink_path": "data/interim/bgcs/Lactobacillus_delbrueckii/6.1.1/GCA_000014405.1/CP000412.1.region001.gbk"
+        },
+        "GCA_000014405.1.gbk": {
+            "record_id": "CP000412.1",
+            "original_id": "CP000412.1",
+            "target_path": "data/interim/antismash/6.1.1/GCA_000014405.1/GCA_000014405.1.gbk",
+            "symlink_path": "data/interim/bgcs/Lactobacillus_delbrueckii/6.1.1/GCA_000014405.1/GCA_000014405.1.gbk"
+        }
+    }
+}

.tests/unit/antismash_overview_gather/data/data/interim/bgcs/Lactobacillus_delbrueckii/6.1.1/GCA_000056065.1/GCA_000056065.1-change_log.json

@@ -0,0 +1,22 @@
+{
+    "GCA_000056065.1": {
+        "CR954253.1.region001.gbk": {
+            "record_id": "CR954253.1",
+            "original_id": "CR954253.1",
+            "target_path": "data/interim/antismash/6.1.1/GCA_000056065.1/CR954253.1.region001.gbk",
+            "symlink_path": "data/interim/bgcs/Lactobacillus_delbrueckii/6.1.1/GCA_000056065.1/CR954253.1.region001.gbk"
+        },
+        "CR954253.1.region002.gbk": {
+            "record_id": "CR954253.1",
+            "original_id": "CR954253.1",
+            "target_path": "data/interim/antismash/6.1.1/GCA_000056065.1/CR954253.1.region002.gbk",
+            "symlink_path": "data/interim/bgcs/Lactobacillus_delbrueckii/6.1.1/GCA_000056065.1/CR954253.1.region002.gbk"
+        },
+        "GCA_000056065.1.gbk": {
+            "record_id": "CR954253.1",
+            "original_id": "CR954253.1",
+            "target_path": "data/interim/antismash/6.1.1/GCA_000056065.1/GCA_000056065.1.gbk",
+            "symlink_path": "data/interim/bgcs/Lactobacillus_delbrueckii/6.1.1/GCA_000056065.1/GCA_000056065.1.gbk"
+        }
+    }
+}

.tests/unit/antismash_overview_gather/data/data/interim/bgcs/Lactobacillus_delbrueckii/6.1.1/GCA_000182835.1/GCA_000182835.1-change_log.json

@@ -0,0 +1,10 @@
+{
+    "GCA_000182835.1": {
+        "GCA_000182835.1.gbk": {
+            "record_id": "CP002342.1",
+            "original_id": "CP002342.1",
+            "target_path": "data/interim/antismash/6.1.1/GCA_000182835.1/GCA_000182835.1.gbk",
+            "symlink_path": "data/interim/bgcs/Lactobacillus_delbrueckii/6.1.1/GCA_000182835.1/GCA_000182835.1.gbk"
+        }
+    }
+}

.tests/unit/antismash_overview_gather/data/data/interim/bgcs/Lactobacillus_delbrueckii/6.1.1/GCA_000191165.1/GCA_000191165.1-change_log.json

@@ -0,0 +1,16 @@
+{
+    "GCA_000191165.1": {
+        "CP000156.1.region001.gbk": {
+            "record_id": "CP000156.1",
+            "original_id": "CP000156.1",
+            "target_path": "data/interim/antismash/6.1.1/GCA_000191165.1/CP000156.1.region001.gbk",
+            "symlink_path": "data/interim/bgcs/Lactobacillus_delbrueckii/6.1.1/GCA_000191165.1/CP000156.1.region001.gbk"
+        },
+        "GCA_000191165.1.gbk": {
+            "record_id": "CP000156.1",
+            "original_id": "CP000156.1",
+            "target_path": "data/interim/antismash/6.1.1/GCA_000191165.1/GCA_000191165.1.gbk",
+            "symlink_path": "data/interim/bgcs/Lactobacillus_delbrueckii/6.1.1/GCA_000191165.1/GCA_000191165.1.gbk"
+        }
+    }
+}

.tests/unit/antismash_overview_gather/data/workflow

@@ -0,0 +1 @@
+../../../../workflow

.tests/unit/antismash_overview_gather/expected/data/processed/Lactobacillus_delbrueckii/tables/df_antismash_6.1.1_bgc.csv

@@ -0,0 +1,5 @@
+bgc_id,genome_id,region,accession,start_pos,end_pos,contig_edge,product,region_length
+CR954253.1.region001,GCA_000056065.1,1.1,CR954253.1,17407,39909,False,['lanthipeptide-class-iii'],22502
+CR954253.1.region002,GCA_000056065.1,1.2,CR954253.1,1745672,1767868,False,['lanthipeptide-class-iv'],22196
+CP000156.1.region001,GCA_000191165.1,1.1,CP000156.1,1767251,1789447,False,['lanthipeptide-class-iv'],22196
+CP000412.1.region001,GCA_000014405.1,1.1,CP000412.1,17283,39785,False,['lanthipeptide-class-iii'],22502