Luigi pipeline sketch

docs/conf.py

@@ -31,6 +31,7 @@
 version = bluesearch.__version__
 
 # -- General configuration ---------------------------------------------------
+suppress_warnings = ["ref.ref"]  # because of luigi.util.requires
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom

docs/source/api/bluesearch.entrypoint.database.rst

@@ -14,6 +14,7 @@ Submodules
    bluesearch.entrypoint.database.parent
    bluesearch.entrypoint.database.parse
    bluesearch.entrypoint.database.parse_mesh_rdf
+   bluesearch.entrypoint.database.run
    bluesearch.entrypoint.database.schemas
    bluesearch.entrypoint.database.topic_extract
    bluesearch.entrypoint.database.topic_filter

docs/source/api/bluesearch.entrypoint.database.run.rst

@@ -0,0 +1,7 @@
+bluesearch.entrypoint.database.run module
+=========================================
+
+.. automodule:: bluesearch.entrypoint.database.run
+   :members:
+   :undoc-members:
+   :show-inheritance:

luigi.cfg

@@ -0,0 +1,44 @@
+;Blue Brain Search is a text mining toolbox focused on scientific use cases.
+;
+;Copyright (C) 2020  Blue Brain Project, EPFL.
+;
+;This program is free software: you can redistribute it and/or modify
+;it under the terms of the GNU Lesser General Public License as published by
+;the Free Software Foundation, either version 3 of the License, or
+;(at your option) any later version.
+;
+;This program is distributed in the hope that it will be useful,
+;but WITHOUT ANY WARRANTY; without even the implied warranty of
+;MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;GNU Lesser General Public License for more details.
+;
+;You should have received a copy of the GNU Lesser General Public License
+;along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+[core]
+autoload_range = true
+log_level = INFO
+local_scheduler = True
+
+[GlobalParams]
+source = pubmed
+
+[DownloadTask]
+from_month = 2021-12
+output_dir = luigi-pipeline
+identifier =
+; emtpy string is considered default value
+
+[TopicExtractTask]
+mesh_topic_db = luigi-pipeline/mesh_topic_db.json
+
+[TopicFilterTask]
+filter_config = luigi-pipeline/filter-config.jsonl
+
+[ConvertPDFTask]
+grobid_host = 0.0.0.0
+grobid_port = 8070
+
+[AddTask]
+db_url = luigi-pipeline/my-db.db
+db_type = sqlite

requirements.txt

@@ -28,6 +28,7 @@ ipython==7.31.1
 ipywidgets==7.6.3
 jupyterlab==3.0.17
 langdetect==1.0.9
+luigi==3.0.3
 mashumaro==3.0
 numpy==1.21.0
 pandas==1.3.0

setup.py

@@ -60,10 +60,11 @@
     "ipywidgets",
     "jupyterlab>=3",
     "langdetect",
-    "numpy>=1.20.1",
-    "pandas>=1",
+    "luigi",
     # Serialization framework on top of dataclasses, e.g. 'Article' to and from JSON.
     "mashumaro>=3.0",
+    "numpy>=1.20.1",
+    "pandas>=1",
     "pg8000",
     "python-dotenv",
     "requests",

src/bluesearch/entrypoint/database/add.py

@@ -124,6 +124,7 @@ def run(
     sentence_mappings = []
 
     for article in articles:
+        logger.info(f"Processing {article.uid}")
 
         article_mapping = {
             "article_id": article.uid,

src/bluesearch/entrypoint/database/parent.py

@@ -14,6 +14,7 @@
     init,
     parse,
     parse_mesh_rdf,
+    run,
     topic_extract,
     topic_filter,
 )
@@ -72,6 +73,11 @@ def main(argv: Sequence[str] | None = None) -> int:
             init_parser=parse.init_parser,
             run=parse.run,
         ),
+        "run": Cmd(
+            help="Run the pipeline.",
+            init_parser=run.init_parser,
+            run=run.run,
+        ),
         "topic-extract": Cmd(
             help="Extract topic of article(s).",
             init_parser=topic_extract.init_parser,