pyproject.toml

[tool.poetry]
name = "agbaye"
version = "0.1.0"
description = "Preprocessing Common Crawl for African Text"
authors = ["theyorubayesian <akin.o.oladipo@gmail.com>"]
readme = "README.md"

[tool.poetry.dependencies]
python = "^3.11"
datatrove = {extras = ["cli", "io", "processing", "s3"], version = "^0.3.0"}
adlfs = "^2024.7.0"
python-dotenv = "^1.0.1"
cleantext = "^1.1.4"

[tool.poetry.group.dev.dependencies]
ipykernel = "^6.29.5"

[tool.poetry.group.afrolid.dependencies]
afrolid = {git = "https://github.com/theyorubayesian/afrolid.git"}
sentencepiece = "^0.2.0"


[tool.poetry.group.mediapipe.dependencies]
mediapipe = "^0.10.20"


[tool.poetry.group.geolid.dependencies]
geolid = "^1.0"


[tool.poetry.group.evaluation.dependencies]
scikit-learn = "^1.6.0"
pycountry = "^24.6.1"


[tool.poetry.group.lid_training.dependencies]
wandb = "^0.19.1"

[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

[tool.poetry.scripts]
agbaye = "agbaye.__main__:main"