yacinecheikh/ai-voice-cloning
1
0
Fork 1
forked from mrq/ai-voice-cloning
Code Pull Requests Activity

actually accumulate derivatives when estimating milestones and final loss by using half of the log

Browse Source
This commit is contained in:
mrq 2023-03-05 14:39:24 +00:00
parent 35225a35da
commit b8a620e8d7
1 changed files with 25 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
src/utils.py
View File

@ -805,20 +805,27 @@ class TrainingState():
self.metrics['loss'].append(f'Loss: {"{:.3f}".format(self.losses[-1]["value"])}') self.metrics['loss'].append(f'Loss: {"{:.3f}".format(self.losses[-1]["value"])}')
if len(self.losses) >= 2: if len(self.losses) >= 2:
# i can probably do a """riemann sum""" to get a better derivative, but the instantaneous one works fine # """riemann sum""" but not really as this is for derivatives and not integrals
d1_loss = self.losses[-1]["value"] deriv = 0
d2_loss = self.losses[-2]["value"] accum_length = len(self.losses)//2 # i *guess* this is fine when you think about it
dloss = d2_loss - d1_loss for i in range(accum_length):
d1_loss = self.losses[-i-1]["value"]
d2_loss = self.losses[-i-2]["value"]
dloss = (d2_loss - d1_loss)
d1_step = self.losses[-1]["step"] d1_step = self.losses[-i-1]["step"]
d2_step = self.losses[-2]["step"] d2_step = self.losses[-i-2]["step"]
dstep = d2_step - d1_step dstep = (d2_step - d1_step)
if dstep == 0:
continue
# don't bother if the loss went up
if True: # dloss < 0:
its_remain = self.its - self.it
inst_deriv = dloss / dstep inst_deriv = dloss / dstep
deriv += inst_deriv
deriv = deriv / accum_length
if deriv != 0: # dloss < 0:
next_milestone = None next_milestone = None
for milestone in self.loss_milestones: for milestone in self.loss_milestones:
if d1_loss > milestone: if d1_loss > milestone:
@ -827,10 +834,13 @@ class TrainingState():
if next_milestone: if next_milestone:
# tfw can do simple calculus but not basic algebra in my head # tfw can do simple calculus but not basic algebra in my head
est_its = (next_milestone - d1_loss) * (dstep / dloss) est_its = (next_milestone - d1_loss) / deriv
print("Estimated iteration to next milestone", est_its)
if est_its >= 0:
self.metrics['loss'].append(f'Est. milestone {next_milestone} in: {int(est_its)}its') self.metrics['loss'].append(f'Est. milestone {next_milestone} in: {int(est_its)}its')
else: else:
est_loss = inst_deriv * its_remain + d1_loss est_loss = inst_deriv * (self.its - self.it) + d1_loss
if est_loss >= 0:
self.metrics['loss'].append(f'Est. final loss: {"{:.3f}".format(est_loss)}') self.metrics['loss'].append(f'Est. final loss: {"{:.3f}".format(est_loss)}')
self.metrics['loss'] = ", ".join(self.metrics['loss']) self.metrics['loss'] = ", ".join(self.metrics['loss'])