deviandice/DL-Art-School
1
0
Fork 0
forked from mrq/DL-Art-School
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tfd3 mods

Browse Source
This commit is contained in:
James Betker 2022-05-27 11:16:26 -06:00
parent bed3df4888
commit 0659fe3d1e
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
codes/models/audio/music/transformer_diffusion3.py
View File

@ -97,6 +97,8 @@ class TransformerDiffusion(nn.Module):
use_rmsnorm=True,
ff_glu=True,
rotary_pos_emb=True,
zero_init_branch_output=True,
ff_mult=1,
)
# Either code_converter or latent_converter is used, depending on what type of conditioning data is fed.
@ -118,6 +120,8 @@ class TransformerDiffusion(nn.Module):
use_rmsnorm=True,
ff_glu=True,
rotary_pos_emb=True,
zero_init_branch_output=True,
ff_mult=1,
)
)
self.latent_fade = nn.Parameter(torch.zeros(1,1,model_channels))
@ -130,6 +134,8 @@ class TransformerDiffusion(nn.Module):
use_rmsnorm=True,
ff_glu=True,
rotary_pos_emb=True,
zero_init_branch_output=True,
ff_mult=1,
)
self.unconditioned_embedding = nn.Parameter(torch.randn(1,1,model_channels))