From d70954266328d2e4f0cd93f28d0200c8a4e530c1 Mon Sep 17 00:00:00 2001 From: clas_benchmark Date: Wed, 10 Feb 2021 03:33:31 +0000 Subject: [PATCH] fix a bug in fp16 training of BERT that will cause non-covergence. --- benchmark/bert/run_pretrain_single.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/bert/run_pretrain_single.py b/benchmark/bert/run_pretrain_single.py index 03430af4f496b..4ab3502d29c63 100644 --- a/benchmark/bert/run_pretrain_single.py +++ b/benchmark/bert/run_pretrain_single.py @@ -238,7 +238,7 @@ def do_train(args): p.name for n, p in model.named_parameters() if not any(nd in n for nd in ["bias", "norm"]) ], - multi_precision=args.use_pure_fp16) + multi_precision=False) if args.use_amp: custom_black_list = (['lookup_table', 'lookup_table_v2'] if args.use_pure_fp16 else None)