mrq/torchscale
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove unused parameters

Browse Source
This commit is contained in:
shumingma 2022-11-29 21:36:03 -08:00
parent 5adbe971cf
commit 7b29d32f03
2 changed files with 2 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
torchscale/architecture/config.py
View File

@ -112,10 +112,10 @@ class DecoderConfig(object):
self.ddp_rank = kwargs.pop("ddp_rank", 0) self.ddp_rank = kwargs.pop("ddp_rank", 0)
if self.deepnorm: if self.deepnorm:
self.encoder_normalize_before = False self.decoder_normalize_before = False
self.subln = False self.subln = False
if self.subln: if self.subln:
self.encoder_normalize_before = True self.decoder_normalize_before = True
self.deepnorm = False self.deepnorm = False
if self.use_xmoe: if self.use_xmoe:
self.moe_normalize_gate_prob_before_dropping = True self.moe_normalize_gate_prob_before_dropping = True

5
torchscale/architecture/decoder.py
View File

@ -92,11 +92,6 @@ class DecoderLayer(nn.Module):
else: else:
self.alpha = 1.0 self.alpha = 1.0
if args.subln:
self.ffn_layernorm = LayerNorm(self.ffn_dim)
else:
self.ffn_layernorm = None
def build_ffn(self, embed_dim, args): def build_ffn(self, embed_dim, args):
return FeedForwardNetwork( return FeedForwardNetwork(
embed_dim, embed_dim,