[taskcluster:warn 2024-05-22T21:21:14.327Z]     index:
[taskcluster:warn 2024-05-22T21:21:14.327Z]       rank: 0
[taskcluster:warn 2024-05-22T21:21:14.327Z]     parent: C50B8ScLS_GtjkiJKFTHdQ
[taskcluster:warn 2024-05-22T21:21:14.327Z]   metadata:
[taskcluster:warn 2024-05-22T21:21:14.327Z]     description: cefilter for ru-en
[taskcluster:warn 2024-05-22T21:21:14.327Z]     name: cefilter-ru-en
[taskcluster:warn 2024-05-22T21:21:14.327Z]     owner: [email protected]
[taskcluster:warn 2024-05-22T21:21:14.327Z]     source: https://github.com/vrigal/firefox-translations-training/blob/6d213f88caf646f94d7922167ab47ecbe3c0424d/taskcluster/kinds/cefilter
[taskcluster:warn 2024-05-22T21:21:14.327Z]   payload:
[taskcluster:warn 2024-05-22T21:21:14.327Z]     artifacts:
[taskcluster:warn 2024-05-22T21:21:14.327Z]     - expires: "2024-08-20T20:54:42.662Z"
[taskcluster:warn 2024-05-22T21:21:14.327Z]       name: public/build
[taskcluster:warn 2024-05-22T21:21:14.327Z]       path: artifact0
[taskcluster:warn 2024-05-22T21:21:14.327Z]       type: directory
[taskcluster:warn 2024-05-22T21:21:14.327Z]     command:
[taskcluster:warn 2024-05-22T21:21:14.327Z]     - - bash
[taskcluster:warn 2024-05-22T21:21:14.327Z]       - -cx
[taskcluster:warn 2024-05-22T21:21:14.327Z]       - |-
[taskcluster:warn 2024-05-22T21:21:14.327Z]         timeout 604800 podman run -t --name taskcontainer --privileged -v "$(pwd)/cache0:/builds/worker/checkouts" --add-host=taskcluster:127.0.0.1 --net=host -e ARTIFACT_EXT -e COMPRESSION_CMD -e FIREFOX_TRANSLATIONS_TRAINING_BASE_REPOSITORY -e FIREFOX_TRANSLATIONS_TRAINING_HEAD_REF -e FIREFOX_TRANSLATIONS_TRAINING_HEAD_REPOSITORY -e FIREFOX_TRANSLATIONS_TRAINING_HEAD_REV -e FIREFOX_TRANSLATIONS_TRAINING_REPOSITORY_TYPE -e HG_STORE_PATH -e MOZ_AUTOMATION -e MOZ_FETCHES -e MOZ_FETCHES_DIR -e MOZ_SCM_LEVEL -e REPOSITORIES -e RUN_ID -e SCCACHE_DISABLE -e SRC -e TASKCLUSTER_CACHES -e TASKCLUSTER_PROXY_URL -e TASKCLUSTER_ROOT_URL -e TASKCLUSTER_UNTRUSTED_CACHES -e TASKCLUSTER_VOLUMES -e TASKCLUSTER_WORKER_LOCATION -e TASK_ID -e TRG -e VCS_PATH docker-archive:dockerimage /usr/local/bin/run-task --firefox_translations_training-checkout=/builds/worker/checkouts/vcs/ -- bash -c '$VCS_PATH/pipeline/cefilter/ce-filter.sh $MOZ_FETCHES_DIR/corpus /builds/worker/artifacts/corpus $MOZ_FETCHES_DIR/scores.txt'
[taskcluster:warn 2024-05-22T21:21:14.327Z]         exit_code=$?
[taskcluster:warn 2024-05-22T21:21:14.327Z]         podman cp taskcontainer:/builds/worker/artifacts artifact0
[taskcluster:warn 2024-05-22T21:21:14.327Z]         podman rm taskcontainer
[taskcluster:warn 2024-05-22T21:21:14.327Z]         exit "${exit_code}"
[taskcluster:warn 2024-05-22T21:21:14.327Z]     env:
[taskcluster:warn 2024-05-22T21:21:14.327Z]       ARTIFACT_EXT: zst
[taskcluster:warn 2024-05-22T21:21:14.327Z]       COMPRESSION_CMD: zstdmt
[taskcluster:warn 2024-05-22T21:21:14.327Z]       FIREFOX_TRANSLATIONS_TRAINING_BASE_REPOSITORY: https://github.com/mozilla/firefox-translations-training
[taskcluster:warn 2024-05-22T21:21:14.327Z]       FIREFOX_TRANSLATIONS_TRAINING_HEAD_REF: publish-comet-metrics
[taskcluster:warn 2024-05-22T21:21:14.327Z]       FIREFOX_TRANSLATIONS_TRAINING_HEAD_REPOSITORY: https://github.com/vrigal/firefox-translations-training
[taskcluster:warn 2024-05-22T21:21:14.327Z]       FIREFOX_TRANSLATIONS_TRAINING_HEAD_REV: 6d213f88caf646f94d7922167ab47ecbe3c0424d
[taskcluster:warn 2024-05-22T21:21:14.327Z]       FIREFOX_TRANSLATIONS_TRAINING_REPOSITORY_TYPE: git
[taskcluster:warn 2024-05-22T21:21:14.327Z]       HG_STORE_PATH: /builds/worker/checkouts/hg-store
[taskcluster:warn 2024-05-22T21:21:14.327Z]       MOZ_AUTOMATION: "1"
[taskcluster:warn 2024-05-22T21:21:14.327Z]       MOZ_FETCHES: '[{"artifact": "public/build/corpus.en.zst", "extract": false, "task":
[taskcluster:warn 2024-05-22T21:21:14.327Z]         "R7_QXie9Sz2ncWWj_8sc8w"}, {"artifact": "public/build/corpus.ru.zst", "extract":
[taskcluster:warn 2024-05-22T21:21:14.327Z]         false, "task": "R7_QXie9Sz2ncWWj_8sc8w"}, {"artifact": "public/build/scores.txt",
[taskcluster:warn 2024-05-22T21:21:14.327Z]         "extract": true, "task": "PMwfKtJUS2aU-uUU87tPnQ"}]'
[taskcluster:warn 2024-05-22T21:21:14.327Z]       MOZ_FETCHES_DIR: fetches
[taskcluster:warn 2024-05-22T21:21:14.327Z]       MOZ_SCM_LEVEL: "1"
[taskcluster:warn 2024-05-22T21:21:14.327Z]       REPOSITORIES: '{"firefox_translations_training": "firefox-translations-training"}'
[taskcluster:warn 2024-05-22T21:21:14.327Z]       SCCACHE_DISABLE: "1"
[taskcluster:warn 2024-05-22T21:21:14.327Z]       SRC: ru
[taskcluster:warn 2024-05-22T21:21:14.327Z]       TASKCLUSTER_CACHES: /builds/worker/checkouts
[taskcluster:warn 2024-05-22T21:21:14.327Z]       TASKCLUSTER_UNTRUSTED_CACHES: "1"
[taskcluster:warn 2024-05-22T21:21:14.327Z]       TASKCLUSTER_VOLUMES: /builds/worker/checkouts
[taskcluster:warn 2024-05-22T21:21:14.327Z]       TRG: en
[taskcluster:warn 2024-05-22T21:21:14.327Z]       VCS_PATH: /builds/worker/checkouts/vcs
[taskcluster:warn 2024-05-22T21:21:14.327Z]     features:
[taskcluster:warn 2024-05-22T21:21:14.327Z]       backingLog: true
[taskcluster:warn 2024-05-22T21:21:14.327Z]       liveLog: true
[taskcluster:warn 2024-05-22T21:21:14.327Z]       taskclusterProxy: true
[taskcluster:warn 2024-05-22T21:21:14.327Z]     logs:
[taskcluster:warn 2024-05-22T21:21:14.327Z]       backing: public/logs/live_backing.log
[taskcluster:warn 2024-05-22T21:21:14.327Z]       live: public/logs/live.log
[taskcluster:warn 2024-05-22T21:21:14.327Z]     maxRunTime: 605700
[taskcluster:warn 2024-05-22T21:21:14.327Z]     mounts:
[taskcluster:warn 2024-05-22T21:21:14.327Z]     - cacheName: translations-level-1-checkouts-v3-7afeb851dd97df8f3607-Q_deBg9LRdWXU21Qhn7s6Q
[taskcluster:warn 2024-05-22T21:21:14.327Z]       directory: cache0
[taskcluster:warn 2024-05-22T21:21:14.327Z]     - content:
[taskcluster:warn 2024-05-22T21:21:14.327Z]         artifact: public/image.tar.zst
[taskcluster:warn 2024-05-22T21:21:14.327Z]         taskId: Q_deBg9LRdWXU21Qhn7s6Q
[taskcluster:warn 2024-05-22T21:21:14.327Z]       file: dockerimage
[taskcluster:warn 2024-05-22T21:21:14.327Z]       format: zst
[taskcluster:warn 2024-05-22T21:21:14.327Z]     onExitStatus:
[taskcluster:warn 2024-05-22T21:21:14.327Z]       purgeCaches:
[taskcluster:warn 2024-05-22T21:21:14.327Z]       - 72
[taskcluster:warn 2024-05-22T21:21:14.327Z]       retry:
[taskcluster:warn 2024-05-22T21:21:14.327Z]       - 72
[taskcluster:warn 2024-05-22T21:21:14.327Z]       - 125
[taskcluster:warn 2024-05-22T21:21:14.327Z]       - 128
[taskcluster:warn 2024-05-22T21:21:14.327Z]   priority: low
[taskcluster:warn 2024-05-22T21:21:14.327Z]   projectId: none
[taskcluster:warn 2024-05-22T21:21:14.327Z]   provisionerId: translations-1
[taskcluster:warn 2024-05-22T21:21:14.327Z]   requires: all-completed
[taskcluster:warn 2024-05-22T21:21:14.327Z]   retries: 5
[taskcluster:warn 2024-05-22T21:21:14.327Z]   routes:
[taskcluster:warn 2024-05-22T21:21:14.327Z]   - index.translations.cache.head.publish-comet-metrics.cefilter.cefilter-ru-en.hash.fdb2f729fcda9dc2c4a311ccbca8b4e28c662bbd4c8a8a6622e8db7830303f4d
[taskcluster:warn 2024-05-22T21:21:14.327Z]   - checks
[taskcluster:warn 2024-05-22T21:21:14.327Z]   schedulerId: translations-level-1
[taskcluster:warn 2024-05-22T21:21:14.327Z]   scopes:
[taskcluster:warn 2024-05-22T21:21:14.327Z]   - generic-worker:cache:translations-level-1-checkouts-v3-7afeb851dd97df8f3607-Q_deBg9LRdWXU21Qhn7s6Q
[taskcluster:warn 2024-05-22T21:21:14.327Z]   tags:
[taskcluster:warn 2024-05-22T21:21:14.327Z]     createdForUser: [email protected]
[taskcluster:warn 2024-05-22T21:21:14.327Z]     kind: cefilter
[taskcluster:warn 2024-05-22T21:21:14.327Z]     label: cefilter-ru-en
[taskcluster:warn 2024-05-22T21:21:14.327Z]     os: linux
[taskcluster:warn 2024-05-22T21:21:14.327Z]     worker-implementation: docker-worker
[taskcluster:warn 2024-05-22T21:21:14.327Z]   taskGroupId: C50B8ScLS_GtjkiJKFTHdQ
[taskcluster:warn 2024-05-22T21:21:14.327Z]   taskQueueId: translations-1/b-linux-large-gcp-d2g-300gb
[taskcluster:warn 2024-05-22T21:21:14.327Z]   workerType: b-linux-large-gcp-d2g-300gb
[taskcluster:warn 2024-05-22T21:21:14.327Z] 
[taskcluster 2024-05-22T21:21:14.666Z] Uploading redirect artifact public/logs/live.log to URL https://firefoxci-websocktunnel.services.mozilla.com/us-west1-a.6740829115498356371.60099/log/XsosTdGeQCew15geub5BOg with mime type "text/plain; charset=utf-8" and expiry 2024-05-29T21:51:14.651Z
[taskcluster 2024-05-22T21:21:15.090Z] [mounts] No existing writable directory cache 'translations-level-1-checkouts-v3-7afeb851dd97df8f3607-Q_deBg9LRdWXU21Qhn7s6Q' - creating caches/H0WDIjROSfyLmMxPHdjNyQ
[taskcluster 2024-05-22T21:21:15.091Z] [mounts] Creating directory /task_171641272440819/cache0
[taskcluster 2024-05-22T21:21:15.099Z] [mounts] Granting task_171641272440819 full control of directory '/task_171641272440819/cache0'
[taskcluster 2024-05-22T21:21:15.100Z] [mounts] Successfully mounted writable directory cache '/task_171641272440819/cache0'
[taskcluster:warn 2024-05-22T21:21:18.249Z] [mounts] No SHA256 specified in task mounts for artifact:Q_deBg9LRdWXU21Qhn7s6Q:public/image.tar.zst - SHA256 from downloaded file downloads/SPqrpMWVRJStFZlFr3u-IA is 20c0bfb9ede9bdea57036ae328d96caaf0223203dea299969fe41279a9b4600d.
[taskcluster 2024-05-22T21:21:18.249Z] [mounts] Creating directory /task_171641272440819
[taskcluster 2024-05-22T21:21:18.257Z] [mounts] Decompressing zst file downloads/SPqrpMWVRJStFZlFr3u-IA to '/task_171641272440819/dockerimage'
[taskcluster 2024-05-22T21:21:19.430Z] [mounts] Granting task_171641272440819 full control of file '/task_171641272440819/dockerimage'
[taskcluster 2024-05-22T21:21:19.431Z] Executing command 0: bash -cx 'timeout 604800 podman run -t --name taskcontainer --privileged -v "$(pwd)/cache0:/builds/worker/checkouts" --add-host=taskcluster:127.0.0.1 --net=host -e ARTIFACT_EXT -e COMPRESSION_CMD -e FIREFOX_TRANSLATIONS_TRAINING_BASE_REPOSITORY -e FIREFOX_TRANSLATIONS_TRAINING_HEAD_REF -e FIREFOX_TRANSLATIONS_TRAINING_HEAD_REPOSITORY -e FIREFOX_TRANSLATIONS_TRAINING_HEAD_REV -e FIREFOX_TRANSLATIONS_TRAINING_REPOSITORY_TYPE -e HG_STORE_PATH -e MOZ_AUTOMATION -e MOZ_FETCHES -e MOZ_FETCHES_DIR -e MOZ_SCM_LEVEL -e REPOSITORIES -e RUN_ID -e SCCACHE_DISABLE -e SRC -e TASKCLUSTER_CACHES -e TASKCLUSTER_PROXY_URL -e TASKCLUSTER_ROOT_URL -e TASKCLUSTER_UNTRUSTED_CACHES -e TASKCLUSTER_VOLUMES -e TASKCLUSTER_WORKER_LOCATION -e TASK_ID -e TRG -e VCS_PATH docker-archive:dockerimage /usr/local/bin/run-task --firefox_translations_training-checkout=/builds/worker/checkouts/vcs/ -- bash -c '\''$VCS_PATH/pipeline/cefilter/ce-filter.sh $MOZ_FETCHES_DIR/corpus /builds/worker/artifacts/corpus $MOZ_FETCHES_DIR/scores.txt'\''
[taskcluster 2024-05-22T21:21:19.431Z] exit_code=$?
[taskcluster 2024-05-22T21:21:19.431Z] podman cp taskcontainer:/builds/worker/artifacts artifact0
[taskcluster 2024-05-22T21:21:19.431Z] podman rm taskcontainer
[taskcluster 2024-05-22T21:21:19.431Z] exit "${exit_code}"'
++ pwd
+ timeout 604800 podman run -t --name taskcontainer --privileged -v /task_171641272440819/cache0:/builds/worker/checkouts --add-host=taskcluster:127.0.0.1 --net=host -e ARTIFACT_EXT -e COMPRESSION_CMD -e FIREFOX_TRANSLATIONS_TRAINING_BASE_REPOSITORY -e FIREFOX_TRANSLATIONS_TRAINING_HEAD_REF -e FIREFOX_TRANSLATIONS_TRAINING_HEAD_REPOSITORY -e FIREFOX_TRANSLATIONS_TRAINING_HEAD_REV -e FIREFOX_TRANSLATIONS_TRAINING_REPOSITORY_TYPE -e HG_STORE_PATH -e MOZ_AUTOMATION -e MOZ_FETCHES -e MOZ_FETCHES_DIR -e MOZ_SCM_LEVEL -e REPOSITORIES -e RUN_ID -e SCCACHE_DISABLE -e SRC -e TASKCLUSTER_CACHES -e TASKCLUSTER_PROXY_URL -e TASKCLUSTER_ROOT_URL -e TASKCLUSTER_UNTRUSTED_CACHES -e TASKCLUSTER_VOLUMES -e TASKCLUSTER_WORKER_LOCATION -e TASK_ID -e TRG -e VCS_PATH docker-archive:dockerimage /usr/local/bin/run-task --firefox_translations_training-checkout=/builds/worker/checkouts/vcs/ -- bash -c '$VCS_PATH/pipeline/cefilter/ce-filter.sh $MOZ_FETCHES_DIR/corpus /builds/worker/artifacts/corpus $MOZ_FETCHES_DIR/scores.txt'
Getting image source signatures
Copying blob sha256:1a84dc453bda5bdb00677e6e599015475a779eb490b8b3158be45af99da083e3
Copying blob sha256:5498e8c22f6996f25ef193ee58617d5b37e2a96decf22e72de13c3b34e147591
Copying blob sha256:50b43097ddf2501adc2f8b21b8685a7508ddd563177028cae61a8ce8cd51a0bb
Copying config sha256:1e62b245ce9b86147f4b5a4cdcf3708c9a763f8cdaa58df95c74f81427d083ab
Writing manifest to image destination
Storing signatures
[setup 2024-05-22T21:21:32.939Z] run-task started in /builds/worker
[setup 2024-05-22T21:21:32.939Z] Invoked by command: --firefox_translations_training-checkout=/builds/worker/checkouts/vcs/ -- bash -c $VCS_PATH/pipeline/cefilter/ce-filter.sh $MOZ_FETCHES_DIR/corpus /builds/worker/artifacts/corpus $MOZ_FETCHES_DIR/scores.txt
[setup 2024-05-22T21:21:32.939Z] Python version: 3.10.12
[cache 2024-05-22T21:21:32.941Z] cache /builds/worker/checkouts is empty; writing requirements: gid=1000 uid=1000 version=1
[volume 2024-05-22T21:21:32.941Z] volume /builds/worker/checkouts is a cache
[setup 2024-05-22T21:21:32.941Z] running as worker:worker
[vcs 2024-05-22T21:21:32.942Z] executing ['git', 'config', '--global', '--add', 'safe.directory', '/builds/worker/checkouts/vcs']
[vcs 2024-05-22T21:21:32.948Z] executing ['git', 'clone', 'https://github.com/mozilla/firefox-translations-training', '/builds/worker/checkouts/vcs']
[vcs 2024-05-22T21:21:32.950Z] Cloning into '/builds/worker/checkouts/vcs'...
[vcs 2024-05-22T21:21:34.421Z] executing ['git', 'fetch', '--no-tags', 'https://github.com/vrigal/firefox-translations-training', 'publish-comet-metrics']
[vcs 2024-05-22T21:21:35.468Z] From https://github.com/vrigal/firefox-translations-training
[vcs 2024-05-22T21:21:35.468Z]  * branch            publish-comet-metrics -> FETCH_HEAD
[vcs 2024-05-22T21:21:35.470Z] executing ['git', 'checkout', '-f', '-B', 'publish-comet-metrics', '6d213f88caf646f94d7922167ab47ecbe3c0424d']
[vcs 2024-05-22T21:21:35.851Z] Switched to a new branch 'publish-comet-metrics'
[vcs 2024-05-22T21:21:35.852Z] executing ['git', 'submodule', 'init']
[vcs 2024-05-22T21:21:35.899Z] Submodule '3rd_party/browsermt-marian-dev' (https://github.com/browsermt/marian-dev) registered for path '3rd_party/browsermt-marian-dev'
[vcs 2024-05-22T21:21:35.953Z] Submodule 'extract-lex' (https://github.com/marian-nmt/extract-lex) registered for path '3rd_party/extract-lex'
[vcs 2024-05-22T21:21:35.982Z] Submodule 'fast_align' (https://github.com/clab/fast_align) registered for path '3rd_party/fast_align'
[vcs 2024-05-22T21:21:36.093Z] Submodule '3rd_party/kenlm' (https://github.com/kpu/kenlm) registered for path '3rd_party/kenlm'
[vcs 2024-05-22T21:21:36.133Z] Submodule '3rd_party/marian-dev' (https://github.com/marian-nmt/marian-dev) registered for path '3rd_party/marian-dev'
[vcs 2024-05-22T21:21:36.167Z] Submodule '3rd_party/preprocess' (https://github.com/kpu/preprocess.git) registered for path '3rd_party/preprocess'
[vcs 2024-05-22T21:21:36.168Z] executing ['git', 'submodule', 'update', '--force']
[vcs 2024-05-22T21:21:36.190Z] Cloning into '/builds/worker/checkouts/vcs/3rd_party/browsermt-marian-dev'...
[vcs 2024-05-22T21:21:37.805Z] Cloning into '/builds/worker/checkouts/vcs/3rd_party/extract-lex'...
[vcs 2024-05-22T21:21:38.193Z] Cloning into '/builds/worker/checkouts/vcs/3rd_party/fast_align'...
[vcs 2024-05-22T21:21:38.603Z] Cloning into '/builds/worker/checkouts/vcs/3rd_party/kenlm'...
[vcs 2024-05-22T21:21:39.642Z] Cloning into '/builds/worker/checkouts/vcs/3rd_party/marian-dev'...
[vcs 2024-05-22T21:21:40.853Z] Cloning into '/builds/worker/checkouts/vcs/3rd_party/preprocess'...
[vcs 2024-05-22T21:21:41.380Z] Submodule path '3rd_party/browsermt-marian-dev': checked out '11c6ae7c46be21ef96ed10c60f28022fa968939f'
[vcs 2024-05-22T21:21:41.393Z] Submodule path '3rd_party/extract-lex': checked out '42fa605b53f32eaf6c6e0b5677255c21c91b3d49'
[vcs 2024-05-22T21:21:41.405Z] Submodule path '3rd_party/fast_align': checked out 'cab1e9aac8d3bb02ff5ae58218d8d225a039fa11'
[vcs 2024-05-22T21:21:41.430Z] Submodule path '3rd_party/kenlm': checked out 'bbf4fc511266c5d4515047055d7bdec659a6e158'
[vcs 2024-05-22T21:21:41.530Z] Submodule path '3rd_party/marian-dev': checked out 'e8a1a2530fb84cbff7383302ebca393e5875c441'
[vcs 2024-05-22T21:21:41.550Z] Submodule path '3rd_party/preprocess': checked out '64307314b4d5a9a0bd529b5c1036b0710d995eec'
[vcs 2024-05-22T21:21:41.550Z] cleaning git checkout...
[vcs 2024-05-22T21:21:41.550Z] executing ['git', 'clean', '-nxdff']
[vcs 2024-05-22T21:21:41.553Z] removing []
[vcs 2024-05-22T21:21:41.553Z] successfully cleaned git checkout!
[vcs 2024-05-22T21:21:41.554Z] TinderboxPrint:<a href='https://github.com/vrigal/firefox-translations-training/commit/6d213f88caf646f94d7922167ab47ecbe3c0424d' title='Built from firefox-translations-training commit 6d213f88caf646f94d7922167ab47ecbe3c0424d'>6d213f88caf646f94d7922167ab47ecbe3c0424d</a>
[setup 2024-05-22T21:21:41.555Z] MOZ_FETCHES_DIR is /builds/worker/fetches
[fetches 2024-05-22T21:21:41.555Z] fetching artifacts
[fetches 2024-05-22T21:21:41.555Z] executing ['/usr/bin/python3', '-u', '/usr/local/bin/fetch-content', 'task-artifacts']
attempt 1/5
Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/R7_QXie9Sz2ncWWj_8sc8w/artifacts/public/build/corpus.en.zst to /builds/worker/fetches/corpus.en.zst
attempt 1/5
Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/R7_QXie9Sz2ncWWj_8sc8w/artifacts/public/build/corpus.ru.zst to /builds/worker/fetches/corpus.ru.zst
Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/R7_QXie9Sz2ncWWj_8sc8w/artifacts/public/build/corpus.en.zst
Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/R7_QXie9Sz2ncWWj_8sc8w/artifacts/public/build/corpus.ru.zst
attempt 1/5
Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/PMwfKtJUS2aU-uUU87tPnQ/artifacts/public/build/scores.txt to /builds/worker/fetches/scores.txt
Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/PMwfKtJUS2aU-uUU87tPnQ/artifacts/public/build/scores.txt
https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/R7_QXie9Sz2ncWWj_8sc8w/artifacts/public/build/corpus.en.zst resolved to 110102 bytes with sha256 9636828dbc3c4d51d8ecb7c588c5528da9123f8c45a61e2fac1a3b2271a9114d in 0.043s
https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/R7_QXie9Sz2ncWWj_8sc8w/artifacts/public/build/corpus.ru.zst resolved to 1027480 bytes with sha256 6e3ed157cd7493ade1322f6ea366226e9a732787cd59d637b207e7f803ae5f67 in 0.102s
https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/PMwfKtJUS2aU-uUU87tPnQ/artifacts/public/build/scores.txt resolved to 169670 bytes with sha256 6dd4026e92eaa1c78549aa46964613716699fd7ad53c26680506d689e6fc31f3 in 0.143s
PERFHERDER_DATA: {"framework": {"name": "build_metrics"}, "suites": [{"name": "fetch_content", "value": 0.14580931800000485, "lowerIsBetter": true, "shouldAlert": false, "subtests": []}]}
[fetches 2024-05-22T21:21:41.770Z] finished fetching artifacts
[task 2024-05-22T21:21:41.771Z] executing ['bash', '-c', '$VCS_PATH/pipeline/cefilter/ce-filter.sh $MOZ_FETCHES_DIR/corpus /builds/worker/artifacts/corpus $MOZ_FETCHES_DIR/scores.txt']
[task 2024-05-22T21:21:41.773Z] + set -euo pipefail
[task 2024-05-22T21:21:41.773Z] + echo '###### Cross entropy filtering'
[task 2024-05-22T21:21:41.773Z] ###### Cross entropy filtering
[task 2024-05-22T21:21:41.773Z] + test -v SRC
[task 2024-05-22T21:21:41.773Z] + test -v TRG
[task 2024-05-22T21:21:41.773Z] + corpus_prefix=/builds/worker/fetches/corpus
[task 2024-05-22T21:21:41.773Z] + output_prefix=/builds/worker/artifacts/corpus
[task 2024-05-22T21:21:41.773Z] + scores=/builds/worker/fetches/scores.txt
[task 2024-05-22T21:21:41.773Z] + COMPRESSION_CMD=zstdmt
[task 2024-05-22T21:21:41.773Z] + ARTIFACT_EXT=zst
[task 2024-05-22T21:21:41.773Z] ++ dirname /builds/worker/checkouts/vcs/pipeline/cefilter/ce-filter.sh
[task 2024-05-22T21:21:41.774Z] + cd /builds/worker/checkouts/vcs/pipeline/cefilter
[task 2024-05-22T21:21:41.774Z] + remove=0.05
[task 2024-05-22T21:21:41.775Z] ++ dirname /builds/worker/artifacts/corpus
[task 2024-05-22T21:21:41.776Z] + output_dir=/builds/worker/artifacts
[task 2024-05-22T21:21:41.776Z] + tmp=/builds/worker/artifacts/tmp
[task 2024-05-22T21:21:41.776Z] + mkdir -p /builds/worker/artifacts/tmp
[task 2024-05-22T21:21:41.777Z] + echo '### Sorting scores'
[task 2024-05-22T21:21:41.777Z] ### Sorting scores
[task 2024-05-22T21:21:41.777Z] + '[' '!' -s /builds/worker/artifacts/tmp/sorted.zst ']'
[task 2024-05-22T21:21:41.778Z] ++ bc
[task 2024-05-22T21:21:41.778Z] ++ cut -f1 -d.
[task 2024-05-22T21:21:41.778Z] +++ grep MemTotal /proc/meminfo
[task 2024-05-22T21:21:41.778Z] +++ awk '{print $2}'
[task 2024-05-22T21:21:41.779Z] ++ echo '264108052*0.9'
[task 2024-05-22T21:21:41.780Z] + buffer_size=237697246
[task 2024-05-22T21:21:41.780Z] + LC_ALL=C
[task 2024-05-22T21:21:41.780Z] + sort -n -k1,1 -S 237697246K -T /builds/worker/artifacts/tmp
[task 2024-05-22T21:21:41.780Z] + zstdmt
[task 2024-05-22T21:21:41.781Z] ++ zstdmt -dc /builds/worker/fetches/corpus.ru.zst
[task 2024-05-22T21:21:41.781Z] + paste /builds/worker/fetches/scores.txt /dev/fd/63 /dev/fd/62
[task 2024-05-22T21:21:41.781Z] ++ zstdmt -dc /builds/worker/fetches/corpus.en.zst
[task 2024-05-22T21:21:41.850Z] + echo '### Cutting the best scored corpus'
[task 2024-05-22T21:21:41.850Z] ### Cutting the best scored corpus
[task 2024-05-22T21:21:41.850Z] + '[' '!' -s /builds/worker/artifacts/tmp/best.zst ']'
[task 2024-05-22T21:21:41.851Z] ++ zstdmt -dc /builds/worker/artifacts/tmp/sorted.zst
[task 2024-05-22T21:21:41.851Z] ++ wc -l
[task 2024-05-22T21:21:41.862Z] + lines=16967
[task 2024-05-22T21:21:41.863Z] ++ echo '16967*0.05'
[task 2024-05-22T21:21:41.863Z] ++ bc
[task 2024-05-22T21:21:41.863Z] ++ cut -f1 -d.
[task 2024-05-22T21:21:41.864Z] + startline=848
[task 2024-05-22T21:21:41.865Z] + zstdmt -dc /builds/worker/artifacts/tmp/sorted.zst
[task 2024-05-22T21:21:41.865Z] + tail -n +848
[task 2024-05-22T21:21:41.865Z] + cut -f2,3
[task 2024-05-22T21:21:41.865Z] + zstdmt
[task 2024-05-22T21:21:41.918Z] + echo '### Writing output corpus'
[task 2024-05-22T21:21:41.918Z] ### Writing output corpus
[task 2024-05-22T21:21:41.918Z] + zstdmt -dc /builds/worker/artifacts/tmp/best.zst
[task 2024-05-22T21:21:41.918Z] + cut -f2
[task 2024-05-22T21:21:41.918Z] + tee /dev/fd/63
[task 2024-05-22T21:21:41.918Z] + zstdmt
[task 2024-05-22T21:21:41.919Z] ++ cut -f1
[task 2024-05-22T21:21:41.919Z] ++ zstdmt
[task 2024-05-22T21:21:41.967Z] + echo '### Deleting tmp dir'
[task 2024-05-22T21:21:41.967Z] ### Deleting tmp dir
[task 2024-05-22T21:21:41.967Z] + rm -rf /builds/worker/artifacts/tmp
[task 2024-05-22T21:21:41.968Z] + echo '###### Done: Cross entropy filtering'
[task 2024-05-22T21:21:41.968Z] ###### Done: Cross entropy filtering
[fetches 2024-05-22T21:21:41.968Z] removing /builds/worker/fetches
[fetches 2024-05-22T21:21:41.969Z] finished
+ exit_code=0
+ podman cp taskcontainer:/builds/worker/artifacts artifact0
+ podman rm taskcontainer
3887353f205d932bd02dc577f73fc389527e3bf20cc1ef87041f67d2928f0ed5
+ exit 0
[taskcluster 2024-05-22T21:21:44.620Z]    Exit Code: 0
[taskcluster 2024-05-22T21:21:44.620Z]    User Time: 8.604813s
[taskcluster 2024-05-22T21:21:44.620Z]  Kernel Time: 3.652786s
[taskcluster 2024-05-22T21:21:44.620Z]    Wall Time: 25.188948797s
[taskcluster 2024-05-22T21:21:44.620Z]       Result: SUCCEEDED
[taskcluster 2024-05-22T21:21:44.621Z] === Task Finished ===
[taskcluster 2024-05-22T21:21:44.621Z] Task Duration: 25.18957229s
[taskcluster 2024-05-22T21:21:44.661Z] Uploading artifact public/build/corpus.ru.zst from file /task_171641272440819/artifact0/corpus.ru.zst with content encoding "identity", mime type "application/zstd" and expiry 2024-08-20T20:54:42.662Z
[taskcluster 2024-05-22T21:21:44.662Z] Uploading artifact public/build/corpus.en.zst from file /task_171641272440819/artifact0/corpus.en.zst with content encoding "identity", mime type "application/zstd" and expiry 2024-08-20T20:54:42.662Z
[taskcluster 2024-05-22T21:21:44.821Z] [mounts] Preserving cache: Moving "/task_171641272440819/cache0" to "caches/H0WDIjROSfyLmMxPHdjNyQ"
[taskcluster 2024-05-22T21:21:44.822Z] [mounts] Denying task_171641272440819 access to 'caches/H0WDIjROSfyLmMxPHdjNyQ'
[taskcluster 2024-05-22T21:21:44.850Z] Uploading link artifact public/logs/live.log to artifact public/logs/live_backing.log with expiry 2024-08-20T20:54:42.662Z