mrq/ai-voice-cloning
31
74
Fork 43
Code Issues 223 Pull Requests 5 Packages Projects Releases Wiki Activity

added new training tunable: loss_text_ce_loss weight, added option to specify source model in case you want to finetune a finetuned model (for example, train a Japanese finetune on a large dataset, then finetune for a specific voice, need to truly validate if it produces usable output), some bug fixes that came up for some reason now and not earlier

Browse Source
This commit is contained in:
mrq 2023-03-01 01:17:38 +00:00
parent 5037752059
commit c2726fa0d4
3 changed files with 62 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
models/.template.yaml
View File

@ -83,7 +83,7 @@ steps:
losses:
text_ce:
type: direct
weight: .01
weight: ${text_ce_lr_weight}
key: loss_text_ce
mel_ce:
type: direct

29
src/utils.py
View File

@ -489,7 +489,6 @@ class TrainingState():
self.losses = []
self.load_losses()
self.cleanup_old(keep=keep_x_past_datasets)
self.spawn_process()
@ -536,6 +535,9 @@ class TrainingState():
def cleanup_old(self, keep=2):
if keep <= 0:
return
if not os.path.isdir(self.dataset_dir):
return
models = sorted([ int(d[:-8]) for d in os.listdir(f'{self.dataset_dir}/models/') if d[-8:] == "_gpt.pth" ])
states = sorted([ int(d[:-6]) for d in os.listdir(f'{self.dataset_dir}/training_state/') if d[-6:] == ".state" ])
@ -554,6 +556,8 @@ class TrainingState():
def parse(self, line, verbose=False, buffer_size=8, keep_x_past_datasets=0, progress=None ):
self.buffer.append(f'{line}')
should_return = False
# rip out iteration info
if not self.training_started:
if line.find('Start training from epoch') >= 0:
@ -654,7 +658,7 @@ class TrainingState():
self.losses['loss_gpt_total'].append(self.info['loss_gpt_total'])
"""
verbose = True
should_return = True
elif line.find('Saving models and training states') >= 0:
self.checkpoint = self.checkpoint + 1
@ -668,8 +672,11 @@ class TrainingState():
self.cleanup_old(keep=keep_x_past_datasets)
if verbose and not self.training_started:
should_return = True
self.buffer = self.buffer[-buffer_size:]
if verbose or not self.training_started:
if should_return:
return "".join(self.buffer)
def run_training(config_path, verbose=False, buffer_size=8, keep_x_past_datasets=0, progress=gr.Progress(track_tqdm=True)):
@ -730,10 +737,10 @@ def stop_training():
print("Killing training process...")
training_state.killed = True
training_state.process.stdout.close()
training_state.process.kill()
training_state.process.terminate()
return_code = training_state.process.wait()
training_state = None
return "Training cancelled"
return f"Training cancelled: {return_code}"
def get_halfp_model_path():
autoregressive_model_path = get_model_path('autoregressive.pth')
@ -828,7 +835,7 @@ EPOCH_SCHEDULE = [ 9, 18, 25, 33 ]
def schedule_learning_rate( iterations ):
return [int(iterations * d) for d in EPOCH_SCHEDULE]
def optimize_training_settings( epochs, learning_rate, learning_rate_schedule, batch_size, mega_batch_factor, print_rate, save_rate, resume_path, half_p, bnb, voice ):
def optimize_training_settings( epochs, learning_rate, text_ce_lr_weight, learning_rate_schedule, batch_size, mega_batch_factor, print_rate, save_rate, resume_path, half_p, bnb, source_model, voice ):
name = f"{voice}-finetune"
dataset_name = f"{voice}-train"
dataset_path = f"./training/{voice}/train.txt"
@ -882,6 +889,7 @@ def optimize_training_settings( epochs, learning_rate, learning_rate_schedule, b
return (
learning_rate,
text_ce_lr_weight,
learning_rate_schedule,
batch_size,
mega_batch_factor,
@ -891,7 +899,10 @@ def optimize_training_settings( epochs, learning_rate, learning_rate_schedule, b
messages
)
def save_training_settings( iterations=None, learning_rate=None, learning_rate_schedule=None, batch_size=None, mega_batch_factor=None, print_rate=None, save_rate=None, name=None, dataset_name=None, dataset_path=None, validation_name=None, validation_path=None, output_name=None, resume_path=None, half_p=None, bnb=None ):
def save_training_settings( iterations=None, learning_rate=None, text_ce_lr_weight=None, learning_rate_schedule=None, batch_size=None, mega_batch_factor=None, print_rate=None, save_rate=None, name=None, dataset_name=None, dataset_path=None, validation_name=None, validation_path=None, output_name=None, resume_path=None, half_p=None, bnb=None, source_model=None ):
if not source_model:
source_model = f"./models/tortoise/autoregressive{'_half' if half_p else ''}.pth"
settings = {
"iterations": iterations if iterations else 500,
"batch_size": batch_size if batch_size else 64,
@ -906,8 +917,10 @@ def save_training_settings( iterations=None, learning_rate=None, learning_rate_s
"validation_name": validation_name if validation_name else "finetune",
"validation_path": validation_path if validation_path else "./training/finetune/train.txt",
"text_ce_lr_weight": text_ce_lr_weight if text_ce_lr_weight else 0.01,
'resume_state': f"resume_state: '{resume_path}'",
'pretrain_model_gpt': f"pretrain_model_gpt: './models/tortoise/autoregressive{'_half' if half_p else ''}.pth'",
'pretrain_model_gpt': f"pretrain_model_gpt: '{source_model}'",
'float16': 'true' if half_p else 'false',
'bitsandbytes': 'true' if bnb else 'false',

67
src/webui.py
View File

@ -198,7 +198,8 @@ def optimize_training_settings_proxy( *args, **kwargs ):
gr.update(value=tup[4]),
gr.update(value=tup[5]),
gr.update(value=tup[6]),
"\n".join(tup[7])
gr.update(value=tup[7]),
"\n".join(tup[8])
)
def import_training_settings_proxy( voice ):
@ -227,6 +228,7 @@ def import_training_settings_proxy( voice ):
batch_size = config['datasets']['train']['batch_size']
mega_batch_factor = config['train']['mega_batch_factor']
iterations = config['train']['niter']
steps_per_iteration = int(lines / batch_size)
@ -234,6 +236,7 @@ def import_training_settings_proxy( voice ):
learning_rate = config['steps']['gpt_train']['optimizer_params']['lr']
text_ce_lr_weight = config['steps']['gpt_train']['losses']['text_ce']['weight']
learning_rate_schedule = [ int(x / steps_per_iteration) for x in config['train']['gen_lr_steps'] ]
@ -243,6 +246,14 @@ def import_training_settings_proxy( voice ):
statedir = f'{outdir}/training_state/' # NOOO STOP MIXING YOUR CASES
resumes = []
resume_path = None
source_model = None
if "pretrain_model_gpt" in config['path']:
source_model = config['path']['pretrain_model_gpt']
elif "resume_state" in config['path']:
resume_path = config['path']['resume_state']
if os.path.isdir(statedir):
resumes = sorted([int(d[:-6]) for d in os.listdir(statedir) if d[-6:] == ".state" ])
@ -250,6 +261,7 @@ def import_training_settings_proxy( voice ):
resume_path = f'{statedir}/{resumes[-1]}.state'
messages.append(f"Latest resume found: {resume_path}")
half_p = config['fp16']
bnb = True
@ -261,6 +273,7 @@ def import_training_settings_proxy( voice ):
return (
epochs,
learning_rate,
text_ce_lr_weight,
learning_rate_schedule,
batch_size,
mega_batch_factor,
@ -269,11 +282,12 @@ def import_training_settings_proxy( voice ):
resume_path,
half_p,
bnb,
source_model,
messages
)
def save_training_settings_proxy( epochs, learning_rate, learning_rate_schedule, batch_size, mega_batch_factor, print_rate, save_rate, resume_path, half_p, bnb, voice ):
def save_training_settings_proxy( epochs, learning_rate, text_ce_lr_weight, learning_rate_schedule, batch_size, mega_batch_factor, print_rate, save_rate, resume_path, half_p, bnb, source_model, voice ):
name = f"{voice}-finetune"
dataset_name = f"{voice}-train"
dataset_path = f"./training/{voice}/train.txt"
@ -299,6 +313,7 @@ def save_training_settings_proxy( epochs, learning_rate, learning_rate_schedule,
iterations=iterations,
batch_size=batch_size,
learning_rate=learning_rate,
text_ce_lr_weight=text_ce_lr_weight,
learning_rate_schedule=learning_rate_schedule,
mega_batch_factor=mega_batch_factor,
print_rate=print_rate,
@ -312,6 +327,7 @@ def save_training_settings_proxy( epochs, learning_rate, learning_rate_schedule,
resume_path=resume_path,
half_p=half_p,
bnb=bnb,
source_model=source_model,
))
return "\n".join(messages)
@ -345,6 +361,12 @@ def setup_gradio():
if args.models_from_local_only:
os.environ['TRANSFORMERS_OFFLINE']='1'
voice_list_with_defaults = get_voice_list(append_defaults=True)
voice_list = get_voice_list()
result_voices = get_voice_list("./results/")
autoregressive_models = get_autoregressive_models()
dataset_list = get_dataset_list()
with gr.Blocks() as ui:
with gr.Tab("Generate"):
with gr.Row():
@ -356,8 +378,7 @@ def setup_gradio():
emotion = gr.Radio( ["Happy", "Sad", "Angry", "Disgusted", "Arrogant", "Custom"], value="Custom", label="Emotion", type="value", interactive=True )
prompt = gr.Textbox(lines=1, label="Custom Emotion + Prompt (if selected)")
voice_list = get_voice_list(append_defaults=True)
voice = gr.Dropdown(choices=voice_list, label="Voice", type="value", value=voice_list[0]) # it'd be very cash money if gradio was able to default to the first value in the list without this shit
voice = gr.Dropdown(choices=voice_list_with_defaults, label="Voice", type="value", value=voice_list_with_defaults[0]) # it'd be very cash money if gradio was able to default to the first value in the list without this shit
mic_audio = gr.Audio( label="Microphone Source", source="microphone", type="filepath" )
voice_latents_chunks = gr.Slider(label="Voice Chunks", minimum=1, maximum=64, value=1, step=1)
with gr.Row():
@ -416,12 +437,10 @@ def setup_gradio():
source_sample = gr.Audio(label="Source Sample", visible=False)
output_audio = gr.Audio(label="Output")
candidates_list = gr.Dropdown(label="Candidates", type="value", visible=False, choices=[""], value="")
# output_pick = gr.Button(value="Select Candidate", visible=False)
def change_candidate( val ):
if not val:
return
print(val)
return val
candidates_list.change(
@ -435,7 +454,6 @@ def setup_gradio():
history_info = gr.Dataframe(label="Results", headers=list(history_headers.keys()))
with gr.Row():
with gr.Column():
result_voices = get_voice_list("./results/")
history_voices = gr.Dropdown(choices=result_voices, label="Voice", type="value", value=result_voices[0] if len(result_voices) > 0 else "")
with gr.Column():
history_results_list = gr.Dropdown(label="Results",type="value", interactive=True, value="")
@ -457,7 +475,7 @@ def setup_gradio():
with gr.Row():
with gr.Column():
dataset_settings = [
gr.Dropdown( get_voice_list(), label="Dataset Source", type="value" ),
gr.Dropdown( choices=voice_list, label="Dataset Source", type="value", value=voice_list[0] if len(voice_list) > 0 else "" ),
gr.Textbox(label="Language", placeholder="English")
]
prepare_dataset_button = gr.Button(value="Prepare")
@ -470,8 +488,12 @@ def setup_gradio():
gr.Number(label="Epochs", value=500, precision=0),
]
with gr.Row():
with gr.Column():
training_settings = training_settings + [
gr.Slider(label="Learning Rate", value=1e-5, minimum=0, maximum=1e-4, step=1e-6),
gr.Slider(label="Text_CE LR Ratio", value=0.01, minimum=0, maximum=1),
]
training_settings = training_settings + [
gr.Slider(label="Learning Rate", value=1e-5, minimum=0, maximum=1e-4, step=1e-6),
gr.Textbox(label="Learning Rate Schedule", placeholder=str(EPOCH_SCHEDULE)),
]
with gr.Row():
@ -489,8 +511,9 @@ def setup_gradio():
]
training_halfp = gr.Checkbox(label="Half Precision", value=args.training_default_halfp)
training_bnb = gr.Checkbox(label="BitsAndBytes", value=args.training_default_bnb)
dataset_list = gr.Dropdown( get_dataset_list(), label="Dataset", type="value" )
training_settings = training_settings + [ training_halfp, training_bnb, dataset_list ]
source_model = gr.Dropdown( choices=autoregressive_models, label="Source Model", type="value", value=autoregressive_models[0] )
dataset_list_dropdown = gr.Dropdown( choices=dataset_list, label="Dataset", type="value", value=dataset_list[0] if len(dataset_list) else "" )
training_settings = training_settings + [ training_halfp, training_bnb, source_model, dataset_list_dropdown ]
with gr.Row():
refresh_dataset_list = gr.Button(value="Refresh Dataset List")
@ -511,9 +534,9 @@ def setup_gradio():
reconnect_training_button = gr.Button(value="Reconnect")
with gr.Column():
training_output = gr.TextArea(label="Console Output", interactive=False, max_lines=8)
verbose_training = gr.Checkbox(label="Verbose Console Output")
verbose_training = gr.Checkbox(label="Verbose Console Output", value=True)
training_buffer_size = gr.Slider(label="Console Buffer Size", minimum=4, maximum=32, value=8)
training_keep_x_past_datasets = gr.Slider(label="Keep X Previous States", minimum=0, maximum=8, value=0)
training_keep_x_past_datasets = gr.Slider(label="Keep X Previous States", minimum=0, maximum=8, value=0, step=1)
training_loss_graph = gr.LinePlot(label="Loss Rates",
x="iteration",
@ -551,7 +574,6 @@ def setup_gradio():
gr.Slider(label="Output Volume", minimum=0, maximum=2, value=args.output_volume),
]
autoregressive_models = get_autoregressive_models()
autoregressive_model_dropdown = gr.Dropdown(choices=autoregressive_models, label="Autoregressive Model", value=args.autoregressive_model if args.autoregressive_model else autoregressive_models[0])
whisper_model_dropdown = gr.Dropdown(["tiny", "tiny.en", "base", "base.en", "small", "small.en", "medium", "medium.en", "large"], label="Whisper Model", value=args.whisper_model)
@ -588,15 +610,6 @@ def setup_gradio():
inputs=autoregressive_model_dropdown,
outputs=None
)
"""
whisper_model_dropdown.change(
fn=update_whisper_model,
inputs=whisper_model_dropdown,
outputs=None
)
"""
# console_output = gr.TextArea(label="Console Output", interactive=False, max_lines=8)
input_settings = [
text,
@ -769,15 +782,15 @@ def setup_gradio():
refresh_dataset_list.click(
lambda: gr.update(choices=get_dataset_list()),
inputs=None,
outputs=dataset_list,
outputs=dataset_list_dropdown,
)
optimize_yaml_button.click(optimize_training_settings_proxy,
inputs=training_settings,
outputs=training_settings[1:8] + [save_yaml_output] #console_output
outputs=training_settings[1:9] + [save_yaml_output] #console_output
)
import_dataset_button.click(import_training_settings_proxy,
inputs=dataset_list,
outputs=training_settings[:10] + [save_yaml_output] #console_output
inputs=dataset_list_dropdown,
outputs=training_settings[:11] + [save_yaml_output] #console_output
)
save_yaml_button.click(save_training_settings_proxy,
inputs=training_settings,