Skip to content

Commit

Permalink
Add argos translator to the comparison
Browse files Browse the repository at this point in the history
marco-c committed Nov 1, 2023
1 parent 619b644 commit ae2fbe7
Showing 4 changed files with 38 additions and 3 deletions.
5 changes: 4 additions & 1 deletion evals/eval/evaluate.py
Original file line number Diff line number Diff line change
@@ -30,7 +30,8 @@

TRANS_ORDER = {'bergamot': 0,
'google': 1,
'microsoft': 2}
'microsoft': 2,
'argos': 3}


def get_dataset_prefix(dataset_name, pair, results_dir):
@@ -118,6 +119,8 @@ def evaluate(pair, set_name, translator, evaluation_engine, gpus, models_dir, re
cmd = f"python3 {os.path.join(HOME_DIR, 'translators', 'google_translate.py')}"
elif translator == 'microsoft':
cmd = f"python3 {os.path.join(HOME_DIR, 'translators', 'microsoft.py')}"
elif translator == 'argos':
cmd = f"python3 {os.path.join(HOME_DIR, 'translators', 'argos.py')}"
else:
raise ValueError(f'Translator is not supported: {translator}')

1 change: 1 addition & 0 deletions evals/requirements.txt
Original file line number Diff line number Diff line change
@@ -8,3 +8,4 @@ pandas==1.1.5
matplotlib==3.4.2
mtdata==0.2.9
unbabel-comet==2.1.1
argostranslate==1.9.1
31 changes: 31 additions & 0 deletions evals/translators/argos.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
import os

import argostranslate.package
import argostranslate.translate
import sys

from tqdm import tqdm


def translate(texts, source, target):
return [argostranslate.translate.translate(text, source, target) for text in tqdm(texts)]


if __name__ == '__main__':
texts = [line.strip() for line in sys.stdin]

source = os.environ['SRC']
target = os.environ['TRG']

argostranslate.package.update_package_index()
available_packages = argostranslate.package.get_available_packages()
package_to_install = next(
filter(
lambda x: x.from_code == source and x.to_code == target, available_packages
)
)
argostranslate.package.install_from_path(package_to_install.download())

translations = translate(texts, source, target)
sys.stdout.write('\n'.join(translations))
sys.stdout.write('\n')
4 changes: 2 additions & 2 deletions scripts/eval.sh
Original file line number Diff line number Diff line change
@@ -3,13 +3,13 @@
set -e

python3 eval/evaluate.py \
--translators=bergamot,microsoft,google \
--translators=bergamot,microsoft,google,argos \
--pairs=all --skip-existing \
--results-dir=/models/evaluation/dev --models-dir=/models/models/dev \
--gpus=1 --evaluation-engine=comet,bleu

python3 eval/evaluate.py \
--translators=bergamot,microsoft,google \
--translators=bergamot,microsoft,google,argos \
--pairs=all --skip-existing \
--results-dir=/models/evaluation/prod --models-dir=/models/models/prod \
--gpus=1 --evaluation-engine=comet,bleu

0 comments on commit ae2fbe7

Please sign in to comment.