Fix transformer kernel call parameters (microsoft#81)

* fix transformer kernel call parameters * remove post-ln modeling
rraminen · Feb 1, 2021 · fde14be · fde14be
1 parent cadc355
commit fde14be
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 6 deletions.
diff --git a/BingBertSquad/turing/nvidia_modeling.py b/BingBertSquad/turing/nvidia_modeling.py
@@ -522,8 +522,8 @@ def __init__(self, config, args):
                 pre_layer_norm=False)
 
             self.layer = nn.ModuleList([
-                copy.deepcopy(DeepSpeedTransformerLayer(i, cuda_config))
-                for i in range(config.num_hidden_layers)
+                copy.deepcopy(DeepSpeedTransformerLayer(cuda_config))
+                for _ in range(config.num_hidden_layers)
             ])
         else:
             layer = BertLayer(config)

diff --git a/BingBertSquad/turing/nvidia_modelingpreln.py b/BingBertSquad/turing/nvidia_modelingpreln.py
@@ -542,8 +542,8 @@ def __init__(self, config, args):
                 pre_layer_norm=True)
 
             self.layer = nn.ModuleList([
-                copy.deepcopy(DeepSpeedTransformerLayer(i, cuda_config))
-                for i in range(config.num_hidden_layers)
+                copy.deepcopy(DeepSpeedTransformerLayer(cuda_config))
+                for _ in range(config.num_hidden_layers)
             ])
         else:
             layer = BertLayer(config)

diff --git a/bing_bert/nvidia/modelingpreln.py b/bing_bert/nvidia/modelingpreln.py
@@ -592,8 +592,8 @@ def __init__(self, config, args, sparse_attention_config=None):
                 stochastic_mode=args.stochastic_mode)
 
             self.layer = nn.ModuleList([
-                copy.deepcopy(DeepSpeedTransformerLayer(i, cuda_config))
-                for i in range(config.num_hidden_layers)
+                copy.deepcopy(DeepSpeedTransformerLayer(cuda_config))
+                for _ in range(config.num_hidden_layers)
             ])
         else:
             layer = BertLayer(config)