mrq/torchscale
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix call to activation_fn

Browse Source
This commit is contained in:
Kashif Rasul 2022-11-29 00:11:38 +01:00
parent be14bc23a1
commit c69aba2a73
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
examples/fairseq/models/bert.py
View File

@ -391,7 +391,7 @@ class ClassificationHead(nn.Module):
x = features[:, 0, :] # take <s> token (equiv. to [CLS])
x = self.dropout(x)
x = self.dense(x)
x = self.activation_fn(x)
x = self.activation_fn(x.float()).as_type(x)
x = self.dropout(x)
x = self.out_proj(x)
return x
@ -418,7 +418,7 @@ class LMHead(nn.Module):
features = features[masked_tokens, :]
x = self.dense(features)
x = self.activation_fn(x)
x = self.activation_fn(x.float()).as_type(x)
x = self.layer_norm(x)
# project back to size of vocabulary with bias
x = F.linear(x, self.weight) + self.bias