[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] save-freq: 25
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] seed: 0
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] sentencepiece-alphas:
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config]   - 0.5
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] sentencepiece-max-lines: 2000000
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] sentencepiece-options: ""
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] sharding: local
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] shuffle: batches
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] shuffle-in-ram: false
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] sigterm: save-and-exit
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] skip: false
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] sqlite: ""
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] sqlite-drop: false
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] sync-freq: 200u
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] sync-sgd: true
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] tempdir: /home/ubuntu/tasks/task_171641325092534/artifacts/tmp
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] throw-on-divergence:
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config]   []
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] tied-embeddings: false
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] tied-embeddings-all: true
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] tied-embeddings-src: false
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] train-embedder-rank:
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config]   []
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] train-sets:
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config]   - stdin
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-aan-activation: swish
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-aan-depth: 2
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-aan-nogate: false
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-decoder-autoreg: rnn
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-decoder-dim-ffn: 0
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-decoder-ffn-depth: 0
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-depth-scaling: false
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-dim-aan: 2048
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-dim-ffn: 1536
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-dropout: 0
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-dropout-attention: 0
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-dropout-ffn: 0
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-ffn-activation: relu
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-ffn-depth: 2
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-guided-alignment-layer: last
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-heads: 8
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-no-affine: false
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-no-bias: false
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-no-projection: false
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-pool: false
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-postprocess: dan
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-postprocess-emb: d
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-postprocess-top: ""
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-preprocess: ""
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-rnn-projection: false
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-tied-layers:
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config]   []
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] transformer-train-position-embeddings: false
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] tsv: true
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] tsv-fields: 3
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] type: transformer
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] ulr: false
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] ulr-dim-emb: 0
[task 2024-05-22T21:29:38.775Z] [2024-05-22 21:29:38] [config] ulr-dropout: 0
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] ulr-keys-vectors: ""
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] ulr-query-vectors: ""
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] ulr-softmax-temperature: 1
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] ulr-trainable-transformation: false
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] unlikelihood-loss: false
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] valid-freq: 50
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] valid-log: /home/ubuntu/tasks/task_171641325092534/artifacts/valid.log
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] valid-max-length: 1000
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] valid-metrics:
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config]   - chrf
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config]   - ce-mean-words
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config]   - bleu-detok
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] valid-mini-batch: 64
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] valid-reset-all: false
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] valid-reset-stalled: true
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] valid-script-args:
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config]   []
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] valid-script-path: ""
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] valid-sets:
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config]   - /home/ubuntu/tasks/task_171641325092534/fetches/devset.ruen.tsv
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] valid-translation-output: /home/ubuntu/tasks/task_171641325092534/artifacts/devset.out
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] vocabs:
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config]   - /home/ubuntu/tasks/task_171641325092534/artifacts/vocab.spm
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config]   - /home/ubuntu/tasks/task_171641325092534/artifacts/vocab.spm
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] word-penalty: 0
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] word-scores: false
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] workspace: 12000
[task 2024-05-22T21:29:38.776Z] [2024-05-22 21:29:38] [config] Model is being created with Marian v1.12.14 2d067af 2024-02-16 11:44:13 -0500
[task 2024-05-22T21:29:38.778Z] [2024-05-22 21:29:38] Using synchronous SGD
[task 2024-05-22T21:29:38.799Z] [tracking INFO] Detected Marian version 1.12
[task 2024-05-22T21:29:38.799Z] [2024-05-22 21:29:38] [comm] Compiled without MPI support. Running as a single process on translations-1-b-linux-v100-gpu-4-300g-ha2jpx-1qkab07lvyi9bxq
[task 2024-05-22T21:29:38.799Z] [2024-05-22 21:29:38] Synced seed 1716413378
[task 2024-05-22T21:29:38.799Z] [2024-05-22 21:29:38] [data] Using word alignments from TSV field no. 2
[task 2024-05-22T21:29:38.799Z] [2024-05-22 21:29:38] [data] Loading SentencePiece vocabulary from file /home/ubuntu/tasks/task_171641325092534/artifacts/vocab.spm
[task 2024-05-22T21:29:38.799Z] [2024-05-22 21:29:38] [data] Setting vocabulary size for input 0 to 1,000
[task 2024-05-22T21:29:38.799Z] [2024-05-22 21:29:38] [data] Loading SentencePiece vocabulary from file /home/ubuntu/tasks/task_171641325092534/artifacts/vocab.spm
[task 2024-05-22T21:29:38.799Z] [2024-05-22 21:29:38] [data] Setting vocabulary size for input 1 to 1,000
[task 2024-05-22T21:29:38.799Z] [2024-05-22 21:29:38] [batching] Collecting statistics for batch fitting with step size 10
[task 2024-05-22T21:29:39.278Z] [2024-05-22 21:29:39] [memory] Extending reserved space to 12032 MB (device gpu0)
[task 2024-05-22T21:29:39.394Z] [2024-05-22 21:29:39] [memory] Extending reserved space to 12032 MB (device gpu1)
[task 2024-05-22T21:29:39.502Z] [2024-05-22 21:29:39] [memory] Extending reserved space to 12032 MB (device gpu2)
[task 2024-05-22T21:29:39.604Z] [2024-05-22 21:29:39] [memory] Extending reserved space to 12032 MB (device gpu3)
[task 2024-05-22T21:29:39.608Z] [2024-05-22 21:29:39] [comm] Using NCCL 2.8.3 for GPU communication
[task 2024-05-22T21:29:39.608Z] [2024-05-22 21:29:39] [comm] Using global sharding
[task 2024-05-22T21:29:39.853Z] [2024-05-22 21:29:39] [comm] NCCLCommunicators constructed successfully
[task 2024-05-22T21:29:39.853Z] [2024-05-22 21:29:39] [training] Using 4 GPUs
[task 2024-05-22T21:29:39.856Z] [2024-05-22 21:29:39] [logits] Applying loss function for 1 factor(s)
[task 2024-05-22T21:29:39.857Z] [2024-05-22 21:29:39] [memory] Reserving 34 MB, device gpu0
[task 2024-05-22T21:29:40.360Z] [2024-05-22 21:29:40] [gpu] 16-bit TensorCores enabled for float32 matrix operations
[task 2024-05-22T21:29:40.641Z] [2024-05-22 21:29:40] [memory] Reserving 34 MB, device gpu0
[task 2024-05-22T21:30:12.534Z] [2024-05-22 21:30:12] [batching] Done. Typical MB size is 360,652 target words
[task 2024-05-22T21:30:12.624Z] [2024-05-22 21:30:12] [memory] Extending reserved space to 12032 MB (device gpu0)
[task 2024-05-22T21:30:12.914Z] [2024-05-22 21:30:12] [memory] Extending reserved space to 12032 MB (device gpu1)
[task 2024-05-22T21:30:12.953Z] [2024-05-22 21:30:12] [memory] Extending reserved space to 12032 MB (device gpu2)
[task 2024-05-22T21:30:12.967Z] [2024-05-22 21:30:12] [memory] Extending reserved space to 12032 MB (device gpu3)
[task 2024-05-22T21:30:12.977Z] [2024-05-22 21:30:12] [comm] Using NCCL 2.8.3 for GPU communication
[task 2024-05-22T21:30:12.977Z] [2024-05-22 21:30:12] [comm] Using global sharding
[task 2024-05-22T21:30:13.153Z] [2024-05-22 21:30:13] [comm] NCCLCommunicators constructed successfully
[task 2024-05-22T21:30:13.153Z] [2024-05-22 21:30:13] [training] Using 4 GPUs
[task 2024-05-22T21:30:13.153Z] [2024-05-22 21:30:13] Training started
[task 2024-05-22T21:35:26.946Z] [2024-05-22 21:35:26] [training] Batches are processed as 1 process(es) x 4 devices/process
[task 2024-05-22T21:35:26.957Z] [2024-05-22 21:35:26] [memory] Reserving 34 MB, device gpu3
[task 2024-05-22T21:35:26.957Z] [2024-05-22 21:35:26] [memory] Reserving 34 MB, device gpu2
[task 2024-05-22T21:35:26.957Z] [2024-05-22 21:35:26] [memory] Reserving 34 MB, device gpu0
[task 2024-05-22T21:35:26.957Z] [2024-05-22 21:35:26] [memory] Reserving 34 MB, device gpu1
[task 2024-05-22T21:35:27.033Z] [2024-05-22 21:35:27] [memory] Reserving 34 MB, device gpu0
[task 2024-05-22T21:35:27.075Z] [2024-05-22 21:35:27] [memory] Reserving 34 MB, device gpu3
[task 2024-05-22T21:35:27.133Z] [2024-05-22 21:35:27] [memory] Reserving 34 MB, device gpu1
[task 2024-05-22T21:35:27.137Z] [2024-05-22 21:35:27] [memory] Reserving 34 MB, device gpu2
[task 2024-05-22T21:35:27.684Z] [2024-05-22 21:35:27] Allocating memory for general optimizer shards
[task 2024-05-22T21:35:27.684Z] [2024-05-22 21:35:27] [memory] Reserving 8 MB, device gpu2
[task 2024-05-22T21:35:27.684Z] [2024-05-22 21:35:27] Parameter type float32, optimization type float32, casting types false
[task 2024-05-22T21:35:27.684Z] [2024-05-22 21:35:27] [memory] Reserving 8 MB, device gpu3
[task 2024-05-22T21:35:27.684Z] [2024-05-22 21:35:27] [memory] Reserving 8 MB, device gpu1
[task 2024-05-22T21:35:27.684Z] [2024-05-22 21:35:27] [memory] Reserving 8 MB, device gpu0
[task 2024-05-22T21:35:27.689Z] [2024-05-22 21:35:27] Allocating memory for Adam-specific shards
[task 2024-05-22T21:35:27.689Z] [2024-05-22 21:35:27] [memory] Reserving 17 MB, device gpu1
[task 2024-05-22T21:35:27.690Z] [2024-05-22 21:35:27] [memory] Reserving 17 MB, device gpu2
[task 2024-05-22T21:35:27.690Z] [2024-05-22 21:35:27] [memory] Reserving 17 MB, device gpu0
[task 2024-05-22T21:35:27.691Z] [2024-05-22 21:35:27] [memory] Reserving 17 MB, device gpu3
[task 2024-05-22T21:35:27.693Z] [2024-05-22 21:35:27] Ep. 1 : Up. 1 : Sen. 1,392 : Cost 2.32636714 : Time 315.09s : 405.37 words/s : gNorm 26.2723 : L.r. 1.8750e-08
[task 2024-05-22T21:35:27.965Z] [2024-05-22 21:35:27] Ep. 1 : Up. 2 : Sen. 3,144 : Cost 4.60387278 : Time 0.27s : 19381.10 words/s : gNorm 28.4477 : L.r. 3.7500e-08
[task 2024-05-22T21:35:28.171Z] [2024-05-22 21:35:28] Ep. 1 : Up. 3 : Sen. 4,536 : Cost 4.95656395 : Time 0.21s : 68166.25 words/s : gNorm 21.6990 : L.r. 5.6250e-08
[task 2024-05-22T21:35:28.433Z] [2024-05-22 21:35:28] Ep. 1 : Up. 4 : Sen. 7,344 : Cost 5.57903576 : Time 0.26s : 64433.61 words/s : gNorm 25.4650 : L.r. 7.5000e-08
[task 2024-05-22T21:35:28.717Z] [2024-05-22 21:35:28] Ep. 1 : Up. 5 : Sen. 8,736 : Cost 2.18980980 : Time 0.28s : 648800.26 words/s : gNorm 25.6550 : L.r. 9.3750e-08
[task 2024-05-22T21:35:29.073Z] [2024-05-22 21:35:29] Ep. 1 : Up. 6 : Sen. 10,128 : Cost 2.25652862 : Time 0.36s : 365097.25 words/s : gNorm 24.9790 : L.r. 1.1250e-07
[task 2024-05-22T21:35:29.267Z] [2024-05-22 21:35:29] Ep. 1 : Up. 7 : Sen. 11,520 : Cost 2.90372634 : Time 0.19s : 140234.96 words/s : gNorm 23.6494 : L.r. 1.3125e-07
[task 2024-05-22T21:35:29.540Z] [2024-05-22 21:35:29] Ep. 1 : Up. 8 : Sen. 16,240 : Cost 5.07914114 : Time 0.27s : 51787.43 words/s : gNorm 24.1244 : L.r. 1.5000e-07
[task 2024-05-22T21:35:29.840Z] [2024-05-22 21:35:29] Ep. 1 : Up. 9 : Sen. 22,928 : Cost 5.99038029 : Time 0.30s : 44599.28 words/s : gNorm 25.6539 : L.r. 1.6875e-07
[task 2024-05-22T21:35:30.062Z] [2024-05-22 21:35:30] Ep. 1 : Up. 10 : Sen. 24,320 : Cost 2.75590730 : Time 0.22s : 207595.40 words/s : gNorm 25.6799 : L.r. 1.8750e-07
[task 2024-05-22T21:35:30.359Z] [2024-05-22 21:35:30] Ep. 1 : Up. 11 : Sen. 25,712 : Cost 2.10363913 : Time 0.30s : 461355.74 words/s : gNorm 25.5933 : L.r. 2.0625e-07
[task 2024-05-22T21:35:30.579Z] [2024-05-22 21:35:30] Ep. 1 : Up. 12 : Sen. 27,104 : Cost 2.61600494 : Time 0.22s : 198149.98 words/s : gNorm 25.0470 : L.r. 2.2500e-07
[task 2024-05-22T21:35:30.835Z] [2024-05-22 21:35:30] Ep. 1 : Up. 13 : Sen. 30,256 : Cost 5.97096825 : Time 0.26s : 24649.72 words/s : gNorm 26.6382 : L.r. 2.4375e-07
[task 2024-05-22T21:35:31.021Z] [2024-05-22 21:35:31] Ep. 1 : Up. 14 : Sen. 31,834 : Cost 1.89467096 : Time 0.19s : 923176.02 words/s : gNorm 26.4731 : L.r. 2.6250e-07
[task 2024-05-22T21:35:31.167Z] [2024-05-22 21:35:31] Ep. 1 : Up. 15 : Sen. 34,559 : Cost 1.79342854 : Time 0.15s : 1086009.04 words/s : gNorm 26.8519 : L.r. 2.8125e-07
[task 2024-05-22T21:35:31.455Z] [2024-05-22 21:35:31] Ep. 1 : Up. 16 : Sen. 40,119 : Cost 5.86007929 : Time 0.29s : 38556.69 words/s : gNorm 27.6832 : L.r. 3.0000e-07
[task 2024-05-22T21:35:31.732Z] [2024-05-22 21:35:31] Ep. 1 : Up. 17 : Sen. 44,839 : Cost 5.97624588 : Time 0.28s : 34074.14 words/s : gNorm 28.5597 : L.r. 3.1875e-07
[task 2024-05-22T21:35:31.999Z] [2024-05-22 21:35:31] Ep. 1 : Up. 18 : Sen. 47,647 : Cost 6.19400263 : Time 0.27s : 42119.13 words/s : gNorm 28.1249 : L.r. 3.3750e-07
[task 2024-05-22T21:35:32.258Z] [2024-05-22 21:35:32] Ep. 1 : Up. 19 : Sen. 54,335 : Cost 5.18647099 : Time 0.26s : 103319.78 words/s : gNorm 27.0671 : L.r. 3.5625e-07
[task 2024-05-22T21:35:32.544Z] [2024-05-22 21:35:32] Ep. 1 : Up. 20 : Sen. 55,727 : Cost 2.08609962 : Time 0.29s : 473392.56 words/s : gNorm 26.8938 : L.r. 3.7500e-07
[task 2024-05-22T21:35:32.803Z] [2024-05-22 21:35:32] Ep. 1 : Up. 21 : Sen. 58,879 : Cost 5.39391088 : Time 0.26s : 36597.41 words/s : gNorm 26.7794 : L.r. 3.9375e-07
[task 2024-05-22T21:35:33.060Z] [2024-05-22 21:35:33] Ep. 1 : Up. 22 : Sen. 60,271 : Cost 6.28942442 : Time 0.26s : 10848.57 words/s : gNorm 27.6237 : L.r. 4.1250e-07
[task 2024-05-22T21:35:33.327Z] [2024-05-22 21:35:33] Ep. 1 : Up. 23 : Sen. 63,423 : Cost 5.28821278 : Time 0.27s : 35329.70 words/s : gNorm 27.4877 : L.r. 4.3125e-07
[task 2024-05-22T21:35:33.620Z] [2024-05-22 21:35:33] Ep. 1 : Up. 24 : Sen. 64,815 : Cost 2.03733945 : Time 0.29s : 541314.76 words/s : gNorm 27.2647 : L.r. 4.5000e-07
[task 2024-05-22T21:35:33.899Z] [2024-05-22 21:35:33] Ep. 1 : Up. 25 : Sen. 71,503 : Cost 5.97861385 : Time 0.28s : 47962.89 words/s : gNorm 27.5935 : L.r. 4.6875e-07
[task 2024-05-22T21:35:33.900Z] [2024-05-22 21:35:33] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz
[task 2024-05-22T21:35:34.205Z] [2024-05-22 21:35:34] Saving Adam parameters
[task 2024-05-22T21:35:34.726Z] [2024-05-22 21:35:34] [training] Saving training checkpoint to /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz and /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz.optimizer.npz
[task 2024-05-22T21:35:35.429Z] [2024-05-22 21:35:35] Ep. 1 : Up. 26 : Sen. 73,791 : Cost 4.89375973 : Time 1.53s : 4488.67 words/s : gNorm 27.6191 : L.r. 4.8750e-07
[task 2024-05-22T21:35:35.637Z] [2024-05-22 21:35:35] Ep. 1 : Up. 27 : Sen. 75,407 : Cost 2.91871190 : Time 0.21s : 130545.88 words/s : gNorm 26.9685 : L.r. 5.0625e-07
[task 2024-05-22T21:35:35.921Z] [2024-05-22 21:35:35] Ep. 1 : Up. 28 : Sen. 79,479 : Cost 5.91804695 : Time 0.28s : 57403.35 words/s : gNorm 26.6178 : L.r. 5.2500e-07
[task 2024-05-22T21:35:36.176Z] [2024-05-22 21:35:36] Ep. 1 : Up. 29 : Sen. 80,871 : Cost 2.15400720 : Time 0.26s : 522782.73 words/s : gNorm 26.3572 : L.r. 5.4375e-07
[task 2024-05-22T21:35:36.446Z] [2024-05-22 21:35:36] Ep. 1 : Up. 30 : Sen. 82,775 : Cost 5.42693043 : Time 0.27s : 28249.00 words/s : gNorm 26.2528 : L.r. 5.6250e-07
[task 2024-05-22T21:35:36.659Z] [2024-05-22 21:35:36] Ep. 1 : Up. 31 : Sen. 84,167 : Cost 3.52399135 : Time 0.21s : 97803.52 words/s : gNorm 25.9221 : L.r. 5.8125e-07
[task 2024-05-22T21:35:36.915Z] [2024-05-22 21:35:36] Ep. 1 : Up. 32 : Sen. 86,695 : Cost 6.17887974 : Time 0.26s : 49377.25 words/s : gNorm 26.1029 : L.r. 6.0000e-07
[task 2024-05-22T21:35:37.032Z] [2024-05-22 21:35:37] Ep. 1 : Up. 33 : Sen. 88,548 : Cost 3.98052430 : Time 0.12s : 159591.15 words/s : gNorm 25.2080 : L.r. 6.1875e-07
[task 2024-05-22T21:35:37.254Z] [2024-05-22 21:35:37] Ep. 1 : Up. 34 : Sen. 89,940 : Cost 2.45857835 : Time 0.22s : 227692.81 words/s : gNorm 24.8931 : L.r. 6.3750e-07
[task 2024-05-22T21:35:37.506Z] [2024-05-22 21:35:37] Ep. 1 : Up. 35 : Sen. 94,660 : Cost 4.68503046 : Time 0.25s : 112270.50 words/s : gNorm 24.6777 : L.r. 6.5625e-07
[task 2024-05-22T21:35:37.832Z] [2024-05-22 21:35:37] Ep. 1 : Up. 36 : Sen. 96,156 : Cost 1.89321482 : Time 0.33s : 544460.02 words/s : gNorm 25.0019 : L.r. 6.7500e-07
[task 2024-05-22T21:35:38.085Z] [2024-05-22 21:35:38] Ep. 1 : Up. 37 : Sen. 101,716 : Cost 5.33657455 : Time 0.25s : 87857.22 words/s : gNorm 24.7370 : L.r. 6.9375e-07
[task 2024-05-22T21:35:38.311Z] [2024-05-22 21:35:38] Ep. 1 : Up. 38 : Sen. 103,108 : Cost 2.24432778 : Time 0.23s : 167384.10 words/s : gNorm 24.6607 : L.r. 7.1250e-07
[task 2024-05-22T21:35:38.586Z] [2024-05-22 21:35:38] Ep. 1 : Up. 39 : Sen. 106,260 : Cost 5.21678495 : Time 0.28s : 34305.75 words/s : gNorm 24.7445 : L.r. 7.3125e-07
[task 2024-05-22T21:35:38.854Z] [2024-05-22 21:35:38] Ep. 1 : Up. 40 : Sen. 110,332 : Cost 4.89338684 : Time 0.27s : 45672.16 words/s : gNorm 24.8632 : L.r. 7.5000e-07
[task 2024-05-22T21:35:39.107Z] [2024-05-22 21:35:39] Ep. 1 : Up. 41 : Sen. 111,828 : Cost 1.94379985 : Time 0.25s : 399605.59 words/s : gNorm 24.8140 : L.r. 7.6875e-07
[task 2024-05-22T21:35:39.240Z] [2024-05-22 21:35:39] Ep. 1 : Up. 42 : Sen. 113,296 : Cost 2.91364074 : Time 0.13s : 154349.82 words/s : gNorm 24.5248 : L.r. 7.8750e-07
[task 2024-05-22T21:35:39.525Z] [2024-05-22 21:35:39] Ep. 1 : Up. 43 : Sen. 119,984 : Cost 5.96252775 : Time 0.28s : 47079.49 words/s : gNorm 24.8525 : L.r. 8.0625e-07
[task 2024-05-22T21:35:39.827Z] [2024-05-22 21:35:39] Ep. 1 : Up. 44 : Sen. 130,696 : Cost 5.70103550 : Time 0.30s : 70762.50 words/s : gNorm 25.0964 : L.r. 8.2500e-07
[task 2024-05-22T21:35:40.046Z] [2024-05-22 21:35:40] Ep. 1 : Up. 45 : Sen. 132,088 : Cost 2.34629035 : Time 0.22s : 159691.33 words/s : gNorm 24.8076 : L.r. 8.4375e-07
[task 2024-05-22T21:35:40.258Z] [2024-05-22 21:35:40] Ep. 1 : Up. 46 : Sen. 134,498 : Cost 1.94651866 : Time 0.21s : 1001913.52 words/s : gNorm 25.0390 : L.r. 8.6250e-07
[task 2024-05-22T21:35:40.462Z] [2024-05-22 21:35:40] Ep. 1 : Up. 47 : Sen. 135,890 : Cost 3.95299387 : Time 0.20s : 88653.63 words/s : gNorm 24.7722 : L.r. 8.8125e-07
[task 2024-05-22T21:35:40.738Z] [2024-05-22 21:35:40] Ep. 1 : Up. 48 : Sen. 139,450 : Cost 5.13041878 : Time 0.28s : 38685.35 words/s : gNorm 24.7982 : L.r. 9.0000e-07
[task 2024-05-22T21:35:41.027Z] [2024-05-22 21:35:41] Ep. 1 : Up. 49 : Sen. 145,010 : Cost 5.83480787 : Time 0.29s : 38475.16 words/s : gNorm 25.1432 : L.r. 9.1875e-07
[task 2024-05-22T21:35:41.297Z] [2024-05-22 21:35:41] Ep. 1 : Up. 50 : Sen. 149,082 : Cost 5.87621784 : Time 0.27s : 30173.92 words/s : gNorm 25.5937 : L.r. 9.3750e-07
[task 2024-05-22T21:35:41.298Z] [2024-05-22 21:35:41] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz
[task 2024-05-22T21:35:41.695Z] [2024-05-22 21:35:41] Saving Adam parameters
[task 2024-05-22T21:35:42.240Z] [2024-05-22 21:35:42] [training] Saving training checkpoint to /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz and /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz.optimizer.npz
[task 2024-05-22T21:35:43.619Z] [2024-05-22 21:35:43] Training finished
[task 2024-05-22T21:35:44.430Z] [2024-05-22 21:35:44] [valid] First sentence's tokens as scored:
[task 2024-05-22T21:35:44.430Z] [2024-05-22 21:35:44] [valid] Decoding validation set with SentencePieceVocab for scoring
[task 2024-05-22T21:35:44.431Z] [2024-05-22 21:35:44] [valid]   Hyp: } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } }
[task 2024-05-22T21:35:44.432Z] [2024-05-22 21:35:44] [valid]   Ref: A M E R I C A N B U S I N E S S M A N E L O N M U S K W I L L S T E P D O W N A S A C H A I R M A N O F T E S L A C O M P A N Y T H A T H E F O U N D E D , F U L F I L L I N G T H E U S A U T H O R I T I E S D E M A N D .
[task 2024-05-22T21:35:54.938Z] [2024-05-22 21:35:54] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz.best-chrf.npz
[task 2024-05-22T21:35:55.085Z] [2024-05-22 21:35:55] [valid] Ep. 1 : Up. 50 : chrf : 0.00217428 : new best
[task 2024-05-22T21:35:55.445Z] [2024-05-22 21:35:55] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz.best-ce-mean-words.npz
[task 2024-05-22T21:35:55.596Z] [2024-05-22 21:35:55] [valid] Ep. 1 : Up. 50 : ce-mean-words : 7.43275 : new best
[task 2024-05-22T21:36:05.768Z] [2024-05-22 21:36:05] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz.best-bleu-detok.npz
[task 2024-05-22T21:36:05.921Z] [2024-05-22 21:36:05] [valid] Ep. 1 : Up. 50 : bleu-detok : 0 : new best
[task 2024-05-22T21:36:05.923Z] [2024-05-22 21:36:05] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz
[task 2024-05-22T21:36:06.316Z] [2024-05-22 21:36:06] Saving Adam parameters
[task 2024-05-22T21:36:06.858Z] [2024-05-22 21:36:06] [training] Saving training checkpoint to /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz and /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz.optimizer.npz
[task 2024-05-22T21:40:45.391Z] [tracking INFO] Successfully parsed 306 lines
[task 2024-05-22T21:40:45.391Z] [tracking INFO] Found 50 training entries
[task 2024-05-22T21:40:45.391Z] [tracking INFO] Found 1 validation entries
[task 2024-05-22T21:40:45.470Z] + cp /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz.best-chrf.npz /home/ubuntu/tasks/task_171641325092534/artifacts/final.model.npz.best-chrf.npz
[task 2024-05-22T21:40:45.510Z] + cp /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz.best-chrf.npz.decoder.yml /home/ubuntu/tasks/task_171641325092534/artifacts/final.model.npz.best-chrf.npz.decoder.yml
[task 2024-05-22T21:40:45.512Z] + echo '### Model training is completed: /home/ubuntu/tasks/task_171641325092534/artifacts'
[task 2024-05-22T21:40:45.512Z] ### Model training is completed: /home/ubuntu/tasks/task_171641325092534/artifacts
[task 2024-05-22T21:40:45.512Z] + echo '###### Done: Training a model'
[task 2024-05-22T21:40:45.512Z] ###### Done: Training a model
[fetches 2024-05-22T21:40:45.529Z] removing /home/ubuntu/tasks/task_171641325092534/fetches
[fetches 2024-05-22T21:40:45.883Z] finished
[taskcluster 2024-05-22T21:40:45.893Z]    Exit Code: 0
[taskcluster 2024-05-22T21:40:45.893Z]    User Time: 39m17.565812s
[taskcluster 2024-05-22T21:40:45.893Z]  Kernel Time: 1m11.247187s
[taskcluster 2024-05-22T21:40:45.893Z]    Wall Time: 13m13.841553608s
[taskcluster 2024-05-22T21:40:45.893Z]       Result: SUCCEEDED
[taskcluster 2024-05-22T21:40:45.893Z] === Task Finished ===
[taskcluster 2024-05-22T21:40:45.893Z] Task Duration: 13m13.843381513s
[taskcluster 2024-05-22T21:40:45.973Z] Uploading artifact public/build/final.model.npz.best-chrf.npz.decoder.yml from file /home/ubuntu/tasks/task_171641325092534/artifacts/final.model.npz.best-chrf.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:45.976Z] Uploading artifact public/build/model.npz.best-bleu-detok.npz from file /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz.best-bleu-detok.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:45.978Z] Uploading artifact public/build/opustrainer.log from file /home/ubuntu/tasks/task_171641325092534/artifacts/opustrainer.log with content encoding "gzip", mime type "text/plain" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:45.982Z] Uploading artifact public/build/final.model.npz.best-chrf.npz from file /home/ubuntu/tasks/task_171641325092534/artifacts/final.model.npz.best-chrf.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:45.988Z] Uploading artifact public/build/model.npz.best-ce-mean-words.npz.decoder.yml from file /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz.best-ce-mean-words.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:46.018Z] Uploading artifact public/build/model.npz.yml from file /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:46.038Z] Uploading artifact public/build/model.npz from file /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:46.038Z] Uploading artifact public/build/model.npz.optimizer.npz from file /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz.optimizer.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:46.038Z] Uploading artifact public/build/model.npz.best-chrf.npz from file /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz.best-chrf.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:46.038Z] Uploading artifact public/build/config.opustrainer.yml.state from file /home/ubuntu/tasks/task_171641325092534/artifacts/config.opustrainer.yml.state with content encoding "gzip", mime type "application/octet-stream" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:46.057Z] Uploading artifact public/build/config.opustrainer.yml from file /home/ubuntu/tasks/task_171641325092534/artifacts/config.opustrainer.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:46.057Z] Uploading artifact public/build/devset.out from file /home/ubuntu/tasks/task_171641325092534/artifacts/devset.out with content encoding "gzip", mime type "application/octet-stream" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:46.064Z] Uploading artifact public/build/model.npz.decoder.yml from file /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:46.105Z] Uploading artifact public/build/model.npz.best-chrf.npz.decoder.yml from file /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz.best-chrf.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:46.105Z] Uploading artifact public/build/model.npz.best-ce-mean-words.npz from file /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz.best-ce-mean-words.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:46.125Z] Uploading artifact public/build/model.npz.best-bleu-detok.npz.decoder.yml from file /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz.best-bleu-detok.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:46.136Z] Uploading artifact public/build/model.npz.progress.yml from file /home/ubuntu/tasks/task_171641325092534/artifacts/model.npz.progress.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:46.168Z] Uploading artifact public/build/train.log from file /home/ubuntu/tasks/task_171641325092534/artifacts/train.log with content encoding "gzip", mime type "text/plain" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:46.168Z] Uploading artifact public/build/valid.log from file /home/ubuntu/tasks/task_171641325092534/artifacts/valid.log with content encoding "gzip", mime type "text/plain" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:46.206Z] Uploading artifact public/build/vocab.spm from file /home/ubuntu/tasks/task_171641325092534/artifacts/vocab.spm with content encoding "gzip", mime type "application/x-source-rpm" and expiry 2024-08-20T20:54:42.883Z
[taskcluster 2024-05-22T21:40:48.154Z] [mounts] Preserving cache: Moving "/home/ubuntu/tasks/task_171641325092534/checkouts" to "/home/ubuntu/caches/eao8UIWmQ6K-KCfbADBYBQ"
[taskcluster 2024-05-22T21:40:48.250Z] Uploading link artifact public/logs/live.log to artifact public/logs/live_backing.log with expiry 2024-08-20T20:54:42.883Z