Merge pull request #90 from marian-nmt/romang/early-stopping-epsilon

Add regression tests for `--early-stopping-epsilon`

tests/training/validation/.gitignore

@@ -16,4 +16,7 @@ compare-trans.bleu
 stop_on_any
 stop_on_all
 stop_on_1st
+eps_stop_on_any
+eps_stop_on_all
+eps_stop_on_1st
 devset.??

tests/training/validation/eps_stop_on_1st.expected

@@ -0,0 +1,16 @@
+[valid] Ep. 1 : Up. 10 : valid-script : 11.2 : new best
+[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61588 : new best
+[valid] Ep. 1 : Up. 20 : valid-script : 11.4 : stalled 1 times (last best: 11.4)
+[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58024 : stalled 1 times (last best: 2.58024)
+[valid] Ep. 1 : Up. 30 : valid-script : 11.6 : stalled 2 times (last best: 11.6)
+[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56749 : stalled 2 times (last best: 2.56749)
+[valid] Ep. 1 : Up. 40 : valid-script : 12.1 : new best
+[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.59487 : stalled 3 times (last best: 2.56749)
+[valid] Ep. 1 : Up. 50 : valid-script : 12.3 : stalled 1 times (last best: 12.3)
+[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.58416 : stalled 4 times (last best: 2.56749)
+[valid] Ep. 1 : Up. 60 : valid-script : 12.5 : stalled 2 times (last best: 12.5)
+[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.60018 : stalled 5 times (last best: 2.56749)
+[valid] Ep. 1 : Up. 70 : valid-script : 12 : stalled 3 times (last best: 12.5)
+[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.6712 : stalled 6 times (last best: 2.56749)
+[valid] Ep. 1 : Up. 80 : valid-script : 12.2 : stalled 4 times (last best: 12.5)
+[valid] Ep. 1 : Up. 80 : ce-mean-words : 2.71746 : stalled 7 times (last best: 2.56749)

tests/training/validation/eps_stop_on_all.expected

@@ -0,0 +1,16 @@
+[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61588 : new best
+[valid] Ep. 1 : Up. 10 : valid-script : 11.2 : new best
+[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58024 : new best
+[valid] Ep. 1 : Up. 20 : valid-script : 11.4 : stalled 1 times (last best: 11.4)
+[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56749 : new best
+[valid] Ep. 1 : Up. 30 : valid-script : 11.6 : stalled 2 times (last best: 11.6)
+[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.59487 : stalled 1 times (last best: 2.56749)
+[valid] Ep. 1 : Up. 40 : valid-script : 12.1 : new best
+[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.58416 : stalled 2 times (last best: 2.56749)
+[valid] Ep. 1 : Up. 50 : valid-script : 12.3 : stalled 1 times (last best: 12.3)
+[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.60018 : stalled 3 times (last best: 2.56749)
+[valid] Ep. 1 : Up. 60 : valid-script : 12.5 : stalled 2 times (last best: 12.5)
+[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.6712 : stalled 4 times (last best: 2.56749)
+[valid] Ep. 1 : Up. 70 : valid-script : 12 : stalled 3 times (last best: 12.5)
+[valid] Ep. 1 : Up. 80 : ce-mean-words : 2.71746 : stalled 5 times (last best: 2.56749)
+[valid] Ep. 1 : Up. 80 : valid-script : 12.2 : stalled 4 times (last best: 12.5)

tests/training/validation/eps_stop_script.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+prefix=eps_stop_script
+
+num1=$(cat $prefix.temp 2>/dev/null || echo 112)
+((num1=2+num1))
+echo $num1 > $prefix.temp
+#(( num1 > 5 )) && num2=2 || num2=1
+num2=$(echo -n "$num1" | cut -c1-2)
+((num1=num1%7))
+echo $num2.$num1

tests/training/validation/test_early_stopping_epsilon_mismatch.sh

@@ -0,0 +1,33 @@
+#!/bin/bash -x
+
+#####################################################################
+# SUMMARY: Check if --early-stopping-epsilon requires the same number of values as --valid-metrics
+# AUTHOR: snukky
+# TAGS: early-stopping early-stopping-epsilon
+#####################################################################
+
+# Exit on error
+set -e
+
+# Remove old artifacts
+rm -rf eps_stop_mismatch eps_stop_mismatch.log
+mkdir -p eps_stop_mismatch
+
+# Test code goes here
+cp $MRT_MODELS/wngt19/model.small.npz eps_stop_on_1st/model.npz
+
+$MRT_MARIAN/marian \
+    --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \
+    -m eps_stop_mismatch/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \
+    -v $MRT_MODELS/wngt19/en-de.{spm,spm} \
+    --disp-freq 5 --valid-freq 10 --after-batches 10 \
+    --valid-metrics bleu ce-mean-words perplexity \
+    --valid-sets $MRT_MODELS/wngt19/newstest2014.{en,de} \
+    --early-stopping 4 --early-stopping-epsilon 0.5 0.1 > eps_stop_mismatch.log 2>&1 || true
+
+# Check error message
+test -e eps_stop_mismatch.log
+grep -q "early.stopping.epsilon.* must have as many values as .*valid.metrics" eps_stop_mismatch.log
+
+# Exit with success code
+exit 0

tests/training/validation/test_early_stopping_epsilon_on_1st.sh

@@ -0,0 +1,42 @@
+#!/bin/bash -x
+
+#####################################################################
+# SUMMARY: Test early stopping with epsilon after stalling on the first validation metric
+# AUTHOR: snukky
+# TAGS: valid valid-script early-stopping early-stopping-epsilon
+#####################################################################
+
+# Exit on error
+set -e
+
+# Test code goes here
+rm -rf eps_stop_on_1st eps_stop_on_1st.log eps_stop_script.temp
+mkdir -p eps_stop_on_1st
+
+# Start training with a fully trained model so that validation metrics do not improve easily
+cp $MRT_MODELS/wngt19/model.small.npz eps_stop_on_1st/model.npz
+
+test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en
+test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de
+
+# Training sides are intentionaly reversed to test early stopping
+$MRT_MARIAN/marian \
+    --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \
+    -m eps_stop_on_1st/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \
+    -v $MRT_MODELS/wngt19/en-de.{spm,spm} \
+    --disp-freq 5 --valid-freq 10 --after-batches 200 \
+    --valid-metrics valid-script ce-mean-words \
+    --valid-script-path ./eps_stop_script.sh \
+    --valid-sets devset.{en,de} \
+    --valid-log eps_stop_on_1st.log \
+    --early-stopping 4 --early-stopping-epsilon 0.4 --early-stopping-on first
+
+test -e eps_stop_on_1st/model.npz
+test -e eps_stop_on_1st/model.npz.yml
+test -e eps_stop_on_1st.log
+
+$MRT_TOOLS/strip-timestamps.sh < eps_stop_on_1st.log | grep '\[valid\]' > eps_stop_on_1st.out
+$MRT_TOOLS/diff-nums.py eps_stop_on_1st.out eps_stop_on_1st.expected -p 0.2 -o eps_stop_on_1st.diff
+
+# Exit with success code
+exit 0

tests/training/validation/test_early_stopping_epsilon_on_all.sh

@@ -0,0 +1,42 @@
+#!/bin/bash -x
+
+#####################################################################
+# SUMMARY: Test early stopping with epsilon after stalling on every validation metrics
+# AUTHOR: snukky
+# TAGS: valid valid-script early-stopping early-stopping-epsilon
+#####################################################################
+
+# Exit on error
+set -e
+
+# Test code goes here
+rm -rf eps_stop_on_all eps_stop_on_all.log eps_stop_script.temp
+mkdir -p eps_stop_on_all
+
+# Start training with a fully trained model so that validation metrics do not improve easily
+cp $MRT_MODELS/wngt19/model.small.npz eps_stop_on_all/model.npz
+
+test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en
+test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de
+
+# Training sides are intentionaly reversed to test early stopping
+$MRT_MARIAN/marian \
+    --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \
+    -m eps_stop_on_all/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \
+    -v $MRT_MODELS/wngt19/en-de.{spm,spm} \
+    --disp-freq 5 --valid-freq 10 --after-batches 200 \
+    --valid-metrics ce-mean-words valid-script \
+    --valid-script-path ./eps_stop_script.sh \
+    --valid-sets devset.{en,de} \
+    --valid-log eps_stop_on_all.log \
+    --early-stopping 4 --early-stopping-epsilon -0.2 0.4 --early-stopping-on all
+
+test -e eps_stop_on_all/model.npz
+test -e eps_stop_on_all/model.npz.yml
+test -e eps_stop_on_all.log
+
+$MRT_TOOLS/strip-timestamps.sh < eps_stop_on_all.log | grep '\[valid\]' > eps_stop_on_all.out
+$MRT_TOOLS/diff-nums.py eps_stop_on_all.out eps_stop_on_all.expected -p 0.2 -o eps_stop_on_all.diff
+
+# Exit with success code
+exit 0