mrq/DL-Art-School
1
3
Fork 5
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tfd3 mods

Browse Source
This commit is contained in:
James Betker 2022-05-27 11:16:26 -06:00
parent bed3df4888
commit 0659fe3d1e
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
codes/models/audio/music/transformer_diffusion3.py
View File

@ -97,6 +97,8 @@ class TransformerDiffusion(nn.Module):
use_rmsnorm=True, use_rmsnorm=True,
ff_glu=True, ff_glu=True,
rotary_pos_emb=True, rotary_pos_emb=True,
zero_init_branch_output=True,
ff_mult=1,
) )
# Either code_converter or latent_converter is used, depending on what type of conditioning data is fed. # Either code_converter or latent_converter is used, depending on what type of conditioning data is fed.
@ -118,6 +120,8 @@ class TransformerDiffusion(nn.Module):
use_rmsnorm=True, use_rmsnorm=True,
ff_glu=True, ff_glu=True,
rotary_pos_emb=True, rotary_pos_emb=True,
zero_init_branch_output=True,
ff_mult=1,
) )
) )
self.latent_fade = nn.Parameter(torch.zeros(1,1,model_channels)) self.latent_fade = nn.Parameter(torch.zeros(1,1,model_channels))
@ -130,6 +134,8 @@ class TransformerDiffusion(nn.Module):
use_rmsnorm=True, use_rmsnorm=True,
ff_glu=True, ff_glu=True,
rotary_pos_emb=True, rotary_pos_emb=True,
zero_init_branch_output=True,
ff_mult=1,
) )
self.unconditioned_embedding = nn.Parameter(torch.randn(1,1,model_channels)) self.unconditioned_embedding = nn.Parameter(torch.randn(1,1,model_channels))