forked from mrq/ai-voice-cloning
oops
This commit is contained in:
parent
d312019d05
commit
cd8702ab0d
|
@ -799,7 +799,7 @@ class TrainingState():
|
||||||
|
|
||||||
self.metrics['loss'] = []
|
self.metrics['loss'] = []
|
||||||
if len(self.losses) > 0:
|
if len(self.losses) > 0:
|
||||||
self.metrics['loss'].append(f'Loss: {"{:3f}".format(self.losses[-1]["value"])}')
|
self.metrics['loss'].append(f'Loss: {"{:.3f}".format(self.losses[-1]["value"])}')
|
||||||
|
|
||||||
if len(self.losses) >= 2:
|
if len(self.losses) >= 2:
|
||||||
# i can probably do a """riemann sum""" to get a better derivative, but the instantaneous one works fine
|
# i can probably do a """riemann sum""" to get a better derivative, but the instantaneous one works fine
|
||||||
|
@ -812,7 +812,7 @@ class TrainingState():
|
||||||
dstep = d2_step - d1_step
|
dstep = d2_step - d1_step
|
||||||
|
|
||||||
# don't bother if the loss went up
|
# don't bother if the loss went up
|
||||||
if dloss < 0:
|
if True; # dloss < 0:
|
||||||
its_remain = self.its - self.it
|
its_remain = self.its - self.it
|
||||||
inst_deriv = dloss / dstep
|
inst_deriv = dloss / dstep
|
||||||
|
|
||||||
|
@ -828,7 +828,7 @@ class TrainingState():
|
||||||
self.metrics['loss'].append(f'Est. milestone {next_milestone} in: {int(est_its)}its')
|
self.metrics['loss'].append(f'Est. milestone {next_milestone} in: {int(est_its)}its')
|
||||||
else:
|
else:
|
||||||
est_loss = inst_deriv * its_remain + d1_loss
|
est_loss = inst_deriv * its_remain + d1_loss
|
||||||
self.metrics['loss'].append(f'Est. final loss: {"{:3f}".format(est_loss)}')
|
self.metrics['loss'].append(f'Est. final loss: {"{:.3f}".format(est_loss)}')
|
||||||
|
|
||||||
self.metrics['loss'] = ", ".join(self.metrics['loss'])
|
self.metrics['loss'] = ", ".join(self.metrics['loss'])
|
||||||
|
|
||||||
|
@ -1289,7 +1289,7 @@ def get_training_list(dir="./training/"):
|
||||||
def do_gc():
|
def do_gc():
|
||||||
gc.collect()
|
gc.collect()
|
||||||
try:
|
try:
|
||||||
trytorch.cuda.empty_cache()
|
torch.cuda.empty_cache()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user