[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] throw-on-divergence:
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config]   []
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] tied-embeddings: false
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] tied-embeddings-all: true
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] tied-embeddings-src: false
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] train-embedder-rank:
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config]   []
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] train-sets:
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config]   - stdin
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-aan-activation: swish
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-aan-depth: 2
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-aan-nogate: false
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-decoder-autoreg: rnn
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-decoder-dim-ffn: 0
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-decoder-ffn-depth: 0
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-depth-scaling: false
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-dim-aan: 2048
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-dim-ffn: 1536
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-dropout: 0
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-dropout-attention: 0
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-dropout-ffn: 0
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-ffn-activation: relu
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-ffn-depth: 2
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-guided-alignment-layer: last
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-heads: 8
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-no-affine: false
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-no-bias: false
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-no-projection: false
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-pool: false
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-postprocess: dan
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-postprocess-emb: d
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-postprocess-top: ""
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-preprocess: ""
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-rnn-projection: false
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config] transformer-tied-layers:
[task 2024-05-24T18:41:10.540Z] [2024-05-24 18:41:10] [config]   []
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] transformer-train-position-embeddings: false
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] tsv: true
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] tsv-fields: 3
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] type: transformer
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] ulr: false
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] ulr-dim-emb: 0
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] ulr-dropout: 0
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] ulr-keys-vectors: ""
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] ulr-query-vectors: ""
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] ulr-softmax-temperature: 1
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] ulr-trainable-transformation: false
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] unlikelihood-loss: false
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] valid-freq: 50
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] valid-log: /home/ubuntu/tasks/task_171657595724115/artifacts/valid.log
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] valid-max-length: 1000
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] valid-metrics:
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config]   - chrf
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config]   - ce-mean-words
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config]   - bleu-detok
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] valid-mini-batch: 16
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] valid-reset-all: false
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] valid-reset-stalled: true
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] valid-script-args:
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config]   []
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] valid-script-path: ""
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] valid-sets:
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config]   - /home/ubuntu/tasks/task_171657595724115/fetches/devset.ruen.tsv
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] valid-translation-output: /home/ubuntu/tasks/task_171657595724115/artifacts/devset.out
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] vocabs:
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config]   - /home/ubuntu/tasks/task_171657595724115/artifacts/vocab.spm
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config]   - /home/ubuntu/tasks/task_171657595724115/artifacts/vocab.spm
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] word-penalty: 0
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] word-scores: false
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] workspace: 12000
[task 2024-05-24T18:41:10.541Z] [2024-05-24 18:41:10] [config] Model is being created with Marian v1.12.14 2d067af 2024-02-16 11:44:13 -0500
[task 2024-05-24T18:41:10.542Z] [2024-05-24 18:41:10] Using synchronous SGD
[task 2024-05-24T18:41:10.564Z] [tracking INFO] Detected Marian version 1.12
[task 2024-05-24T18:41:10.564Z] [2024-05-24 18:41:10] [comm] Compiled without MPI support. Running as a single process on translations-1-b-linux-v100-gpu-4-1tb-uelot1xks8st0rkrt0thlw
[task 2024-05-24T18:41:10.564Z] [2024-05-24 18:41:10] Synced seed 1716576070
[task 2024-05-24T18:41:10.564Z] [2024-05-24 18:41:10] [data] Using word alignments from TSV field no. 2
[task 2024-05-24T18:41:10.564Z] [2024-05-24 18:41:10] [data] Loading SentencePiece vocabulary from file /home/ubuntu/tasks/task_171657595724115/artifacts/vocab.spm
[task 2024-05-24T18:41:10.564Z] [2024-05-24 18:41:10] [data] Setting vocabulary size for input 0 to 1,000
[task 2024-05-24T18:41:10.564Z] [2024-05-24 18:41:10] [data] Loading SentencePiece vocabulary from file /home/ubuntu/tasks/task_171657595724115/artifacts/vocab.spm
[task 2024-05-24T18:41:10.564Z] [2024-05-24 18:41:10] [data] Setting vocabulary size for input 1 to 1,000
[task 2024-05-24T18:41:10.564Z] [2024-05-24 18:41:10] [batching] Collecting statistics for batch fitting with step size 10
[task 2024-05-24T18:41:10.980Z] [2024-05-24 18:41:10] [memory] Extending reserved space to 12032 MB (device gpu0)
[task 2024-05-24T18:41:11.096Z] [2024-05-24 18:41:11] [memory] Extending reserved space to 12032 MB (device gpu1)
[task 2024-05-24T18:41:11.199Z] [2024-05-24 18:41:11] [memory] Extending reserved space to 12032 MB (device gpu2)
[task 2024-05-24T18:41:11.305Z] [2024-05-24 18:41:11] [memory] Extending reserved space to 12032 MB (device gpu3)
[task 2024-05-24T18:41:11.310Z] [2024-05-24 18:41:11] [comm] Using NCCL 2.8.3 for GPU communication
[task 2024-05-24T18:41:11.310Z] [2024-05-24 18:41:11] [comm] Using global sharding
[task 2024-05-24T18:41:11.550Z] [2024-05-24 18:41:11] [comm] NCCLCommunicators constructed successfully
[task 2024-05-24T18:41:11.550Z] [2024-05-24 18:41:11] [training] Using 4 GPUs
[task 2024-05-24T18:41:11.553Z] [2024-05-24 18:41:11] [logits] Applying loss function for 1 factor(s)
[task 2024-05-24T18:41:11.554Z] [2024-05-24 18:41:11] [memory] Reserving 34 MB, device gpu0
[task 2024-05-24T18:41:12.165Z] [2024-05-24 18:41:12] [gpu] 16-bit TensorCores enabled for float32 matrix operations
[task 2024-05-24T18:41:12.369Z] [2024-05-24 18:41:12] [memory] Reserving 34 MB, device gpu0
[task 2024-05-24T18:41:44.095Z] [2024-05-24 18:41:44] [batching] Done. Typical MB size is 721,304 target words
[task 2024-05-24T18:41:44.183Z] [2024-05-24 18:41:44] [memory] Extending reserved space to 12032 MB (device gpu0)
[task 2024-05-24T18:41:44.471Z] [2024-05-24 18:41:44] [memory] Extending reserved space to 12032 MB (device gpu1)
[task 2024-05-24T18:41:44.522Z] [2024-05-24 18:41:44] [memory] Extending reserved space to 12032 MB (device gpu2)
[task 2024-05-24T18:41:44.535Z] [2024-05-24 18:41:44] [memory] Extending reserved space to 12032 MB (device gpu3)
[task 2024-05-24T18:41:44.545Z] [2024-05-24 18:41:44] [comm] Using NCCL 2.8.3 for GPU communication
[task 2024-05-24T18:41:44.545Z] [2024-05-24 18:41:44] [comm] Using global sharding
[task 2024-05-24T18:41:44.702Z] [2024-05-24 18:41:44] [comm] NCCLCommunicators constructed successfully
[task 2024-05-24T18:41:44.703Z] [2024-05-24 18:41:44] [training] Using 4 GPUs
[task 2024-05-24T18:41:44.703Z] [2024-05-24 18:41:44] [training] Initializing model weights with pre-trained model /home/ubuntu/tasks/task_171657595724115/fetches/final.model.npz.best-chrf.npz
[task 2024-05-24T18:41:44.880Z] [2024-05-24 18:41:44] No checkpoint found, parameters reloaded from last inference model
[task 2024-05-24T18:41:44.880Z] [2024-05-24 18:41:44] Training started
[taskcluster 2024-05-24T18:56:17.132Z] [taskcluster-proxy] Successfully refreshed taskcluster-proxy credentials: task-client/WDfc8cAMRcW01YqWhmeY_g/0/on/us-central1-a/864168159160918225/until/1716578177.082
[task 2024-05-24T18:58:26.104Z] [2024-05-24 18:58:26] [training] Batches are processed as 1 process(es) x 4 devices/process
[task 2024-05-24T18:58:26.116Z] [2024-05-24 18:58:26] [memory] Reserving 34 MB, device gpu3
[task 2024-05-24T18:58:26.116Z] [2024-05-24 18:58:26] [memory] Reserving 34 MB, device gpu1
[task 2024-05-24T18:58:26.116Z] [2024-05-24 18:58:26] [memory] Reserving 34 MB, device gpu2
[task 2024-05-24T18:58:26.116Z] [2024-05-24 18:58:26] [memory] Reserving 34 MB, device gpu0
[task 2024-05-24T18:58:26.195Z] [2024-05-24 18:58:26] [memory] Reserving 34 MB, device gpu0
[task 2024-05-24T18:58:26.238Z] [2024-05-24 18:58:26] [memory] Reserving 34 MB, device gpu2
[task 2024-05-24T18:58:26.267Z] [2024-05-24 18:58:26] [memory] Reserving 34 MB, device gpu3
[task 2024-05-24T18:58:26.298Z] [2024-05-24 18:58:26] [memory] Reserving 34 MB, device gpu1
[task 2024-05-24T18:58:26.473Z] [2024-05-24 18:58:26] Quantizing the model to 8-bits
[task 2024-05-24T18:58:26.473Z] [2024-05-24 18:58:26] Quantizing the model to 8-bits
[task 2024-05-24T18:58:26.473Z] [2024-05-24 18:58:26] Quantizing the model to 8-bits
[task 2024-05-24T18:58:26.473Z] [2024-05-24 18:58:26] [memory] Reserving 34 MB, device gpu2
[task 2024-05-24T18:58:26.473Z] [2024-05-24 18:58:26] Quantizing the model to 8-bits
[task 2024-05-24T18:58:26.473Z] [2024-05-24 18:58:26] [memory] Reserving 34 MB, device gpu1
[task 2024-05-24T18:58:26.473Z] [2024-05-24 18:58:26] [memory] Reserving 34 MB, device gpu0
[task 2024-05-24T18:58:26.473Z] [2024-05-24 18:58:26] [memory] Reserving 34 MB, device gpu3
[task 2024-05-24T18:58:26.475Z] [2024-05-24 18:58:26] [memory] Reserving 4 B, device gpu1
[task 2024-05-24T18:58:26.475Z] [2024-05-24 18:58:26] [memory] Reserving 4 B, device gpu2
[task 2024-05-24T18:58:26.476Z] [2024-05-24 18:58:26] [memory] Reserving 4 B, device gpu0
[task 2024-05-24T18:58:26.476Z] [2024-05-24 18:58:26] [memory] Reserving 4 B, device gpu3
[task 2024-05-24T18:58:26.993Z] [2024-05-24 18:58:26] Parameter type float32, optimization type float32, casting types false
[task 2024-05-24T18:58:26.993Z] [2024-05-24 18:58:26] Allocating memory for general optimizer shards
[task 2024-05-24T18:58:26.993Z] [2024-05-24 18:58:26] [memory] Reserving 8 MB, device gpu0
[task 2024-05-24T18:58:26.993Z] [2024-05-24 18:58:26] [memory] Reserving 8 MB, device gpu3
[task 2024-05-24T18:58:26.993Z] [2024-05-24 18:58:26] [memory] Reserving 8 MB, device gpu2
[task 2024-05-24T18:58:26.993Z] [2024-05-24 18:58:26] [memory] Reserving 8 MB, device gpu1
[task 2024-05-24T18:58:26.998Z] [2024-05-24 18:58:26] Allocating memory for Adam-specific shards
[task 2024-05-24T18:58:26.998Z] [2024-05-24 18:58:26] [memory] Reserving 17 MB, device gpu1
[task 2024-05-24T18:58:26.999Z] [2024-05-24 18:58:26] [memory] Reserving 17 MB, device gpu0
[task 2024-05-24T18:58:26.999Z] [2024-05-24 18:58:26] [memory] Reserving 17 MB, device gpu3
[task 2024-05-24T18:58:26.999Z] [2024-05-24 18:58:26] [memory] Reserving 17 MB, device gpu2
[task 2024-05-24T18:58:27.012Z] [2024-05-24 18:58:27] Ep. 1 : Up. 1 : Sen. 4,576 : Cost 1.65629613 : Time 1002.85s : 606.88 words/s : gNorm 23.8148 : L.r. 1.8750e-08
[task 2024-05-24T18:58:28.713Z] [2024-05-24 18:58:28] Ep. 1 : Up. 2 : Sen. 9,152 : Cost 1.66741812 : Time 1.70s : 365182.47 words/s : gNorm 22.9303 : L.r. 3.7500e-08
[task 2024-05-24T18:58:29.271Z] [2024-05-24 18:58:29] Ep. 1 : Up. 3 : Sen. 15,456 : Cost 1.71706700 : Time 0.56s : 928722.32 words/s : gNorm 22.6656 : L.r. 5.6250e-08
[task 2024-05-24T18:58:29.805Z] [2024-05-24 18:58:29] Ep. 1 : Up. 4 : Sen. 18,240 : Cost 1.68716276 : Time 0.53s : 1021356.04 words/s : gNorm 23.3146 : L.r. 7.5000e-08
[task 2024-05-24T18:58:30.255Z] [2024-05-24 18:58:30] Ep. 1 : Up. 5 : Sen. 22,400 : Cost 1.78912628 : Time 0.45s : 768284.00 words/s : gNorm 23.9455 : L.r. 9.3750e-08
[task 2024-05-24T18:58:30.879Z] [2024-05-24 18:58:30] Ep. 1 : Up. 6 : Sen. 26,208 : Cost 1.56067073 : Time 0.62s : 963328.68 words/s : gNorm 22.9985 : L.r. 1.1250e-07
[task 2024-05-24T18:58:31.181Z] [2024-05-24 18:58:31] Ep. 1 : Up. 7 : Sen. 29,200 : Cost 2.01530910 : Time 0.30s : 495876.21 words/s : gNorm 22.7971 : L.r. 1.3125e-07
[task 2024-05-24T18:58:31.669Z] [2024-05-24 18:58:31] Ep. 1 : Up. 8 : Sen. 32,704 : Cost 1.51657856 : Time 0.49s : 804320.26 words/s : gNorm 22.6159 : L.r. 1.5000e-07
[task 2024-05-24T18:58:32.174Z] [2024-05-24 18:58:32] Ep. 1 : Up. 9 : Sen. 36,864 : Cost 1.59509599 : Time 0.50s : 1171439.99 words/s : gNorm 22.5656 : L.r. 1.6875e-07
[task 2024-05-24T18:58:32.791Z] [2024-05-24 18:58:32] Ep. 1 : Up. 10 : Sen. 40,368 : Cost 1.64011335 : Time 0.62s : 925188.29 words/s : gNorm 22.5699 : L.r. 1.8750e-07
[task 2024-05-24T18:58:33.416Z] [2024-05-24 18:58:33] Ep. 1 : Up. 11 : Sen. 43,360 : Cost 1.63268292 : Time 0.62s : 858282.33 words/s : gNorm 22.1563 : L.r. 2.0625e-07
[task 2024-05-24T18:58:34.060Z] [2024-05-24 18:58:34] Ep. 1 : Up. 12 : Sen. 47,520 : Cost 1.63892448 : Time 0.64s : 922712.61 words/s : gNorm 21.9944 : L.r. 2.2500e-07
[task 2024-05-24T18:58:34.563Z] [2024-05-24 18:58:34] Ep. 1 : Up. 13 : Sen. 51,328 : Cost 1.78240550 : Time 0.50s : 898075.63 words/s : gNorm 22.0882 : L.r. 2.4375e-07
[task 2024-05-24T18:58:35.052Z] [2024-05-24 18:58:35] Ep. 1 : Up. 14 : Sen. 55,136 : Cost 1.72047007 : Time 0.49s : 754113.63 words/s : gNorm 22.1901 : L.r. 2.6250e-07
[task 2024-05-24T18:58:35.455Z] [2024-05-24 18:58:35] Ep. 1 : Up. 15 : Sen. 58,581 : Cost 1.63421190 : Time 0.40s : 1111378.10 words/s : gNorm 22.3651 : L.r. 2.8125e-07
[task 2024-05-24T18:58:35.781Z] [2024-05-24 18:58:35] Ep. 1 : Up. 16 : Sen. 62,912 : Cost 2.07741261 : Time 0.33s : 654010.73 words/s : gNorm 22.1141 : L.r. 3.0000e-07
[task 2024-05-24T18:58:36.256Z] [2024-05-24 18:58:36] Ep. 1 : Up. 17 : Sen. 66,416 : Cost 1.89768863 : Time 0.48s : 847022.88 words/s : gNorm 21.9513 : L.r. 3.1875e-07
[task 2024-05-24T18:58:36.637Z] [2024-05-24 18:58:36] Ep. 1 : Up. 18 : Sen. 69,648 : Cost 1.95999062 : Time 0.38s : 539187.64 words/s : gNorm 21.5832 : L.r. 3.3750e-07
[task 2024-05-24T18:58:37.213Z] [2024-05-24 18:58:37] Ep. 1 : Up. 19 : Sen. 74,704 : Cost 1.64716053 : Time 0.58s : 1089696.15 words/s : gNorm 21.5824 : L.r. 3.5625e-07
[task 2024-05-24T18:58:37.832Z] [2024-05-24 18:58:37] Ep. 1 : Up. 20 : Sen. 77,488 : Cost 1.63388872 : Time 0.62s : 895507.18 words/s : gNorm 21.9896 : L.r. 3.7500e-07
[task 2024-05-24T18:58:38.375Z] [2024-05-24 18:58:38] Ep. 1 : Up. 21 : Sen. 81,296 : Cost 1.62078023 : Time 0.54s : 1046423.00 words/s : gNorm 21.6925 : L.r. 3.9375e-07
[task 2024-05-24T18:58:38.978Z] [2024-05-24 18:58:38] Ep. 1 : Up. 22 : Sen. 84,080 : Cost 1.66540194 : Time 0.60s : 880836.47 words/s : gNorm 21.6177 : L.r. 4.1250e-07
[task 2024-05-24T18:58:39.503Z] [2024-05-24 18:58:39] Ep. 1 : Up. 23 : Sen. 89,887 : Cost 1.66035700 : Time 0.52s : 1084122.51 words/s : gNorm 22.0554 : L.r. 4.3125e-07
[task 2024-05-24T18:58:40.033Z] [2024-05-24 18:58:40] Ep. 1 : Up. 24 : Sen. 93,119 : Cost 1.47262037 : Time 0.53s : 1054680.30 words/s : gNorm 22.0280 : L.r. 4.5000e-07
[task 2024-05-24T18:58:40.676Z] [2024-05-24 18:58:40] Ep. 1 : Up. 25 : Sen. 95,903 : Cost 1.80503678 : Time 0.64s : 833647.98 words/s : gNorm 22.0364 : L.r. 4.6875e-07
[task 2024-05-24T18:58:40.677Z] [2024-05-24 18:58:40] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz
[task 2024-05-24T18:58:40.955Z] [2024-05-24 18:58:40] Saving Adam parameters
[task 2024-05-24T18:58:41.439Z] [2024-05-24 18:58:41] [training] Saving training checkpoint to /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz and /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz.optimizer.npz
[task 2024-05-24T18:58:42.183Z] [2024-05-24 18:58:42] Ep. 1 : Up. 26 : Sen. 98,687 : Cost 1.82919204 : Time 1.51s : 98998.74 words/s : gNorm 21.9090 : L.r. 4.8750e-07
[task 2024-05-24T18:58:42.735Z] [2024-05-24 18:58:42] Ep. 1 : Up. 27 : Sen. 101,471 : Cost 1.62191486 : Time 0.55s : 988137.21 words/s : gNorm 22.0908 : L.r. 5.0625e-07
[task 2024-05-24T18:58:43.298Z] [2024-05-24 18:58:43] Ep. 1 : Up. 28 : Sen. 104,463 : Cost 1.60259044 : Time 0.56s : 962302.38 words/s : gNorm 22.2129 : L.r. 5.2500e-07
[task 2024-05-24T18:58:43.713Z] [2024-05-24 18:58:43] Ep. 1 : Up. 29 : Sen. 107,695 : Cost 1.84273160 : Time 0.42s : 604394.36 words/s : gNorm 22.4492 : L.r. 5.4375e-07
[task 2024-05-24T18:58:44.371Z] [2024-05-24 18:58:44] Ep. 1 : Up. 30 : Sen. 113,999 : Cost 1.77138770 : Time 0.66s : 959657.28 words/s : gNorm 22.3861 : L.r. 5.6250e-07
[task 2024-05-24T18:58:44.963Z] [2024-05-24 18:58:44] Ep. 1 : Up. 31 : Sen. 117,231 : Cost 1.78488398 : Time 0.59s : 933769.41 words/s : gNorm 22.2190 : L.r. 5.8125e-07
[task 2024-05-24T18:58:45.310Z] [2024-05-24 18:58:45] Ep. 1 : Up. 32 : Sen. 120,463 : Cost 1.86313331 : Time 0.35s : 628592.02 words/s : gNorm 22.1814 : L.r. 6.0000e-07
[task 2024-05-24T18:58:45.797Z] [2024-05-24 18:58:45] Ep. 1 : Up. 33 : Sen. 125,198 : Cost 1.72381735 : Time 0.49s : 1233464.06 words/s : gNorm 22.2181 : L.r. 6.1875e-07
[task 2024-05-24T18:58:46.324Z] [2024-05-24 18:58:46] Ep. 1 : Up. 34 : Sen. 127,982 : Cost 1.73304033 : Time 0.53s : 676853.73 words/s : gNorm 22.0972 : L.r. 6.3750e-07
[task 2024-05-24T18:58:46.469Z] [2024-05-24 18:58:46] Ep. 1 : Up. 35 : Sen. 128,621 : Cost 6.26875019 : Time 0.14s : 4416.30 words/s : gNorm 24.0870 : L.r. 6.5625e-07
[task 2024-05-24T18:58:47.052Z] [2024-05-24 18:58:47] Ep. 1 : Up. 36 : Sen. 132,781 : Cost 1.64375985 : Time 0.58s : 955666.99 words/s : gNorm 23.9903 : L.r. 6.7500e-07
[task 2024-05-24T18:58:47.639Z] [2024-05-24 18:58:47] Ep. 1 : Up. 37 : Sen. 135,565 : Cost 1.63101947 : Time 0.59s : 916304.76 words/s : gNorm 23.8354 : L.r. 6.9375e-07
[task 2024-05-24T18:58:48.201Z] [2024-05-24 18:58:48] Ep. 1 : Up. 38 : Sen. 140,141 : Cost 1.65290105 : Time 0.56s : 846700.61 words/s : gNorm 23.8144 : L.r. 7.1250e-07
[task 2024-05-24T18:58:48.822Z] [2024-05-24 18:58:48] Ep. 1 : Up. 39 : Sen. 146,341 : Cost 1.70787144 : Time 0.62s : 1089550.53 words/s : gNorm 23.7026 : L.r. 7.3125e-07
[task 2024-05-24T18:58:49.546Z] [2024-05-24 18:58:49] Ep. 1 : Up. 40 : Sen. 149,573 : Cost 1.54906654 : Time 0.72s : 797521.71 words/s : gNorm 23.5228 : L.r. 7.5000e-07
[task 2024-05-24T18:58:49.986Z] [2024-05-24 18:58:49] Ep. 1 : Up. 41 : Sen. 152,805 : Cost 1.65053630 : Time 0.44s : 744053.86 words/s : gNorm 23.6075 : L.r. 7.6875e-07
[task 2024-05-24T18:58:50.531Z] [2024-05-24 18:58:50] Ep. 1 : Up. 42 : Sen. 156,965 : Cost 1.95463383 : Time 0.55s : 570032.91 words/s : gNorm 23.6684 : L.r. 7.8750e-07
[task 2024-05-24T18:58:51.088Z] [2024-05-24 18:58:51] Ep. 1 : Up. 43 : Sen. 160,773 : Cost 1.58636367 : Time 0.56s : 1035202.69 words/s : gNorm 23.5384 : L.r. 8.0625e-07
[task 2024-05-24T18:58:51.667Z] [2024-05-24 18:58:51] Ep. 1 : Up. 44 : Sen. 165,349 : Cost 1.72433424 : Time 0.58s : 1042998.29 words/s : gNorm 23.3976 : L.r. 8.2500e-07
[task 2024-05-24T18:58:52.303Z] [2024-05-24 18:58:52] Ep. 1 : Up. 45 : Sen. 168,581 : Cost 1.61337936 : Time 0.64s : 874218.68 words/s : gNorm 23.3370 : L.r. 8.4375e-07
[task 2024-05-24T18:58:52.789Z] [2024-05-24 18:58:52] Ep. 1 : Up. 46 : Sen. 172,085 : Cost 1.67020166 : Time 0.49s : 859460.66 words/s : gNorm 23.2908 : L.r. 8.6250e-07
[task 2024-05-24T18:58:53.286Z] [2024-05-24 18:58:53] Ep. 1 : Up. 47 : Sen. 175,317 : Cost 1.64685023 : Time 0.50s : 847767.74 words/s : gNorm 23.2471 : L.r. 8.8125e-07
[task 2024-05-24T18:58:53.814Z] [2024-05-24 18:58:53] Ep. 1 : Up. 48 : Sen. 182,437 : Cost 1.73199177 : Time 0.53s : 1214464.81 words/s : gNorm 23.2947 : L.r. 9.0000e-07
[task 2024-05-24T18:58:54.404Z] [2024-05-24 18:58:54] Ep. 1 : Up. 49 : Sen. 188,053 : Cost 1.85914493 : Time 0.59s : 657384.16 words/s : gNorm 23.2777 : L.r. 9.1875e-07
[task 2024-05-24T18:58:54.736Z] [2024-05-24 18:58:54] Ep. 1 : Up. 50 : Sen. 190,837 : Cost 3.77723384 : Time 0.33s : 41263.49 words/s : gNorm 23.0633 : L.r. 9.3750e-07
[task 2024-05-24T18:58:54.737Z] [2024-05-24 18:58:54] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz
[task 2024-05-24T18:58:55.131Z] [2024-05-24 18:58:55] Saving Adam parameters
[task 2024-05-24T18:58:55.632Z] [2024-05-24 18:58:55] [training] Saving training checkpoint to /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz and /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz.optimizer.npz
[task 2024-05-24T18:58:57.001Z] [2024-05-24 18:58:57] Training finished
[task 2024-05-24T18:58:57.484Z] [2024-05-24 18:58:57] [valid] First sentence's tokens as scored:
[task 2024-05-24T18:58:57.484Z] [2024-05-24 18:58:57] [valid] Decoding validation set with SentencePieceVocab for scoring
[task 2024-05-24T18:58:57.485Z] [2024-05-24 18:58:57] [valid]   Hyp: o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s o u s
[task 2024-05-24T18:58:57.486Z] [2024-05-24 18:58:57] [valid]   Ref: A N Z H I : D Y U P I N , C H A N C E L L O R , U D A L Y , B E L O R U K O V , S A V I C H E V , K U L I K , R A B I U ( G L E B O V , 8 4 ) , G I G O L A E V , T C H A I K O V S K Y , P O N C E ( O N D O U A , 8 0 ) , D O L G O V ( A K H Y A D O V , 6 9 ) .
[task 2024-05-24T18:59:18.432Z] [2024-05-24 18:59:18] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz.best-chrf.npz
[task 2024-05-24T18:59:18.558Z] [2024-05-24 18:59:18] [valid] Ep. 1 : Up. 50 : chrf : 0.233722 : new best
[task 2024-05-24T18:59:19.309Z] [2024-05-24 18:59:19] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz.best-ce-mean-words.npz
[task 2024-05-24T18:59:19.452Z] [2024-05-24 18:59:19] [valid] Ep. 1 : Up. 50 : ce-mean-words : 7.59496 : new best
[task 2024-05-24T18:59:38.242Z] [2024-05-24 18:59:38] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz.best-bleu-detok.npz
[task 2024-05-24T18:59:38.386Z] [2024-05-24 18:59:38] [valid] Ep. 1 : Up. 50 : bleu-detok : 0 : new best
[task 2024-05-24T18:59:38.388Z] [2024-05-24 18:59:38] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz
[task 2024-05-24T18:59:38.750Z] [2024-05-24 18:59:38] Saving Adam parameters
[task 2024-05-24T18:59:39.240Z] [2024-05-24 18:59:39] [training] Saving training checkpoint to /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz and /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz.optimizer.npz
[taskcluster 2024-05-24T19:13:17.190Z] [taskcluster-proxy] Successfully refreshed taskcluster-proxy credentials: task-client/WDfc8cAMRcW01YqWhmeY_g/0/on/us-central1-a/864168159160918225/until/1716579197.133
[task 2024-05-24T19:15:35.140Z] [tracking INFO] Successfully parsed 315 lines
[task 2024-05-24T19:15:35.140Z] [tracking INFO] Found 50 training entries
[task 2024-05-24T19:15:35.140Z] [tracking INFO] Found 1 validation entries
[task 2024-05-24T19:15:35.223Z] + cp /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz.best-chrf.npz /home/ubuntu/tasks/task_171657595724115/artifacts/final.model.npz.best-chrf.npz
[task 2024-05-24T19:15:35.258Z] + cp /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz.best-chrf.npz.decoder.yml /home/ubuntu/tasks/task_171657595724115/artifacts/final.model.npz.best-chrf.npz.decoder.yml
[task 2024-05-24T19:15:35.260Z] + echo '### Model training is completed: /home/ubuntu/tasks/task_171657595724115/artifacts'
[task 2024-05-24T19:15:35.260Z] ### Model training is completed: /home/ubuntu/tasks/task_171657595724115/artifacts
[task 2024-05-24T19:15:35.260Z] + echo '###### Done: Training a model'
[task 2024-05-24T19:15:35.260Z] ###### Done: Training a model
[fetches 2024-05-24T19:15:35.277Z] removing /home/ubuntu/tasks/task_171657595724115/fetches
[fetches 2024-05-24T19:15:35.641Z] finished
[taskcluster 2024-05-24T19:15:35.651Z]    Exit Code: 0
[taskcluster 2024-05-24T19:15:35.651Z]    User Time: 2h1m27.78972s
[taskcluster 2024-05-24T19:15:35.651Z]  Kernel Time: 2m16.007474s
[taskcluster 2024-05-24T19:15:35.651Z]    Wall Time: 35m30.643792079s
[taskcluster 2024-05-24T19:15:35.651Z]       Result: SUCCEEDED
[taskcluster 2024-05-24T19:15:35.651Z] === Task Finished ===
[taskcluster 2024-05-24T19:15:35.652Z] Task Duration: 35m30.645896711s
[taskcluster 2024-05-24T19:15:35.747Z] Uploading artifact public/build/valid.log from file /home/ubuntu/tasks/task_171657595724115/artifacts/valid.log with content encoding "gzip", mime type "text/plain" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.750Z] Uploading artifact public/build/model.npz.best-ce-mean-words.npz.decoder.yml from file /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz.best-ce-mean-words.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.757Z] Uploading artifact public/build/vocab.spm from file /home/ubuntu/tasks/task_171657595724115/artifacts/vocab.spm with content encoding "gzip", mime type "application/x-source-rpm" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.759Z] Uploading artifact public/build/final.model.npz.best-chrf.npz.decoder.yml from file /home/ubuntu/tasks/task_171657595724115/artifacts/final.model.npz.best-chrf.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.761Z] Uploading artifact public/build/model.npz.best-chrf.npz.decoder.yml from file /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz.best-chrf.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.762Z] Uploading artifact public/build/model.npz from file /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.784Z] Uploading artifact public/build/model.npz.yml from file /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.793Z] Uploading artifact public/build/model.npz.optimizer.npz from file /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz.optimizer.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.793Z] Uploading artifact public/build/model.npz.decoder.yml from file /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.794Z] Uploading artifact public/build/config.opustrainer.yml from file /home/ubuntu/tasks/task_171657595724115/artifacts/config.opustrainer.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.796Z] Uploading artifact public/build/model.npz.best-chrf.npz from file /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz.best-chrf.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.820Z] Uploading artifact public/build/model.npz.progress.yml from file /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz.progress.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.820Z] Uploading artifact public/build/final.model.npz.best-chrf.npz from file /home/ubuntu/tasks/task_171657595724115/artifacts/final.model.npz.best-chrf.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.820Z] Uploading artifact public/build/config.opustrainer.yml.state from file /home/ubuntu/tasks/task_171657595724115/artifacts/config.opustrainer.yml.state with content encoding "gzip", mime type "application/octet-stream" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.820Z] Uploading artifact public/build/opustrainer.log from file /home/ubuntu/tasks/task_171657595724115/artifacts/opustrainer.log with content encoding "gzip", mime type "text/plain" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.820Z] Uploading artifact public/build/devset.out from file /home/ubuntu/tasks/task_171657595724115/artifacts/devset.out with content encoding "gzip", mime type "application/octet-stream" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.820Z] Uploading artifact public/build/train.log from file /home/ubuntu/tasks/task_171657595724115/artifacts/train.log with content encoding "gzip", mime type "text/plain" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.820Z] Uploading artifact public/build/model.npz.best-bleu-detok.npz.decoder.yml from file /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz.best-bleu-detok.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.862Z] Uploading artifact public/build/model.npz.best-bleu-detok.npz from file /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz.best-bleu-detok.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:35.947Z] Uploading artifact public/build/model.npz.best-ce-mean-words.npz from file /home/ubuntu/tasks/task_171657595724115/artifacts/model.npz.best-ce-mean-words.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2024-08-22T16:48:59.299Z
[taskcluster 2024-05-24T19:15:38.214Z] [mounts] Preserving cache: Moving "/home/ubuntu/tasks/task_171657595724115/checkouts" to "/home/ubuntu/caches/TRlwWgzYRoKQ6zNzQsuSeQ"
[taskcluster 2024-05-24T19:15:38.285Z] Uploading link artifact public/logs/live.log to artifact public/logs/live_backing.log with expiry 2024-08-22T16:48:59.299Z