mrq/torchscale
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #5 from kashif/typo

Browse Source 
fix typo
This commit is contained in:
Shuming Ma 2022-11-29 18:00:13 +08:00 committed by GitHub
commit 5adbe971cf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
examples/fairseq/models/bert.py
View File

@ -391,7 +391,7 @@ class ClassificationHead(nn.Module):
x = features[:, 0, :] # take <s> token (equiv. to [CLS])
x = self.dropout(x)
x = self.dense(x)
x = self.activation_fn(x.float()).as_type(x)
x = self.activation_fn(x.float()).type_as(x)
x = self.dropout(x)
x = self.out_proj(x)
return x
@ -418,7 +418,7 @@ class LMHead(nn.Module):
features = features[masked_tokens, :]
x = self.dense(features)
x = self.activation_fn(x.float()).as_type(x)
x = self.activation_fn(x.float()).type_as(x)
x = self.layer_norm(x)
# project back to size of vocabulary with bias
x = F.linear(x, self.weight) + self.bias

2
torchscale/component/feedforward_network.py
View File

@ -121,7 +121,7 @@ class FeedForwardNetwork(nn.Module):
x_shape = x.shape
x = x.reshape(-1, x.size(-1))
x = self.fc1(x)
x = self.activation_fn(x.float()).as_type(x)
x = self.activation_fn(x.float()).type_as(x)
x = self.activation_dropout_module(x)
if self.ffn_layernorm is not None:
x = self.ffn_layernorm(x)