Skip to content

Commit

Permalink
gelu -> quick_gelu in hlb_cifar (tinygrad#3147)
Browse files Browse the repository at this point in the history
89 -> 86 seconds, same eval acc
  • Loading branch information
chenyuxyz authored Jan 16, 2024
1 parent ec5a212 commit b9d4705
Showing 1 changed file with 3 additions and 3 deletions.
6 changes: 3 additions & 3 deletions examples/hlb_cifar10.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,13 @@ def __call__(self, x):
x = x.float()
x = self.norm1(x)
x = x.cast(dtypes.default_float)
x = x.gelu()
x = x.quick_gelu()
residual = x
x = self.conv2(x)
x = x.float()
x = self.norm2(x)
x = x.cast(dtypes.default_float)
x = x.gelu()
x = x.quick_gelu()

return x + residual

Expand All @@ -64,7 +64,7 @@ def __init__(self, W):
self.whitening = W
self.net = [
nn.Conv2d(12, 32, kernel_size=1, bias=False),
lambda x: x.gelu(),
lambda x: x.quick_gelu(),
ConvGroup(32, 64),
ConvGroup(64, 256),
ConvGroup(256, 512),
Expand Down

0 comments on commit b9d4705

Please sign in to comment.