deviandice/DL-Art-School
1
0
Fork 0
forked from mrq/DL-Art-School
Code Issues Pull Requests Packages Projects Releases Wiki Activity

one more adjustment

Browse Source
This commit is contained in:
James Betker 2022-06-11 08:01:46 -06:00
parent df0cdf1a4f
commit 41170f97e9
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
codes/models/audio/music/transformer_diffusion9.py
View File

@ -43,8 +43,7 @@ class TimestepRotaryEmbedSequential(nn.Sequential, TimestepBlock):
class DietAttentionBlock(TimestepBlock): class DietAttentionBlock(TimestepBlock):
def __init__(self, in_dim, dim, heads, dropout): def __init__(self, in_dim, dim, heads, dropout):
super().__init__() super().__init__()
self.proj = nn.Linear(in_dim, dim) self.proj = nn.Linear(in_dim, dim, bias=False)
self.proj.bias.data.zero_()
self.rms_scale_norm = RMSScaleShiftNorm(dim, bias=False) self.rms_scale_norm = RMSScaleShiftNorm(dim, bias=False)
self.attn = Attention(dim, heads=heads, dim_head=dim//heads, causal=False, dropout=dropout) self.attn = Attention(dim, heads=heads, dim_head=dim//heads, causal=False, dropout=dropout)
self.ff = FeedForward(dim, in_dim, mult=1, dropout=dropout, zero_init_output=True) self.ff = FeedForward(dim, in_dim, mult=1, dropout=dropout, zero_init_output=True)