From c956d81baf903d45096c5137cbfa48c2601c1253 Mon Sep 17 00:00:00 2001 From: mrq Date: Thu, 2 Mar 2023 01:35:12 +0000 Subject: [PATCH] added button to just load a training set's loss information, added installing broncotc/bitsandbytes-rocm when running setup-rocm.sh --- setup-cuda.bat | 6 +++--- setup-cuda.sh | 19 +++++++++++-------- setup-directml.bat | 9 +++++---- setup-rocm.sh | 22 ++++++++++++++-------- src/utils.py | 24 +++++++++++++++++------- src/webui.py | 36 +++++++++++++++++++++++++----------- start.bat | 1 - train.bat | 4 ++-- update-force.bat | 14 +++++++++++++- update-force.sh | 15 ++++++++++++++- update.bat | 14 +------------- update.sh | 10 ---------- 12 files changed, 105 insertions(+), 69 deletions(-) diff --git a/setup-cuda.bat b/setup-cuda.bat index 297640c..f071d2b 100755 --- a/setup-cuda.bat +++ b/setup-cuda.bat @@ -7,12 +7,12 @@ python -m pip install --upgrade pip python -m pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu117 python -m pip install -r .\dlas\requirements.txt python -m pip install -r .\tortoise-tts\requirements.txt -python -m pip install -r .\requirements.txt python -m pip install -e .\tortoise-tts\ +python -m pip install -r .\requirements.txt xcopy .\dlas\bitsandbytes_windows\* .\venv\Lib\site-packages\bitsandbytes\. /Y xcopy .\dlas\bitsandbytes_windows\cuda_setup\* .\venv\Lib\site-packages\bitsandbytes\cuda_setup\. /Y xcopy .\dlas\bitsandbytes_windows\nn\* .\venv\Lib\site-packages\bitsandbytes\nn\. /Y -deactivate -pause \ No newline at end of file +pause +deactivate \ No newline at end of file diff --git a/setup-cuda.sh b/setup-cuda.sh index 5901a1d..fd47ce7 100755 --- a/setup-cuda.sh +++ b/setup-cuda.sh @@ -1,14 +1,17 @@ #!/bin/bash +# get local dependencies git submodule init git submodule update --remote - +# setup venv python3 -m venv venv source ./venv/bin/activate -python3 -m pip install --upgrade pip +python3 -m pip install --upgrade pip # just to be safe # CUDA -pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu116 -python3 -m pip install -r ./dlas/requirements.txt -python3 -m pip install -r ./tortoise-tts/requirements.txt -python3 -m pip install -r ./requirements.txt -python3 -m pip install -e ./tortoise-tts/ -deactivate +pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu117 +# install requirements +python3 -m pip install -r ./dlas/requirements.txt # instal DLAS requirements +python3 -m pip install -r ./tortoise-tts/requirements.txt # install TorToiSe requirements +python3 -m pip install -e ./tortoise-tts/ # install TorToiSe +python3 -m pip install -r ./requirements.txt # install local requirements + +deactivate \ No newline at end of file diff --git a/setup-directml.bat b/setup-directml.bat index 2cc193e..1ec13ea 100755 --- a/setup-directml.bat +++ b/setup-directml.bat @@ -4,10 +4,11 @@ git submodule update --remote python -m venv venv call .\venv\Scripts\activate.bat python -m pip install --upgrade pip -python -m pip install torch torchvision torchaudio torch-directml==0.1.13.1.dev230119 +python -m pip install torch torchvision torchaudio torch-directml python -m pip install -r .\dlas\requirements.txt python -m pip install -r .\tortoise-tts\requirements.txt -python -m pip install -r .\requirements.txt python -m pip install -e .\tortoise-tts\ -deactivate -pause \ No newline at end of file +python -m pip install -r .\requirements.txt + +pause +deactivate \ No newline at end of file diff --git a/setup-rocm.sh b/setup-rocm.sh index 53c0a12..e94ee55 100755 --- a/setup-rocm.sh +++ b/setup-rocm.sh @@ -1,14 +1,20 @@ #!/bin/bash +# get local dependencies git submodule init git submodule update --remote - +# setup venv python3 -m venv venv source ./venv/bin/activate -python3 -m pip install --upgrade pip +python3 -m pip install --upgrade pip # just to be safe # ROCM -pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/rocm5.1.1 # 5.2 does not work for me desu -python3 -m pip install -r ./dlas/requirements.txt -python3 -m pip install -r ./tortoise-tts/requirements.txt -python3 -m pip install -r ./requirements.txt -python3 -m pip install -e ./tortoise-tts/ -deactivate +pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/rocm5.1.1 # 5.2 does not work for me desu +# install requirements +python3 -m pip install -r ./dlas/requirements.txt # instal DLAS requirements +python3 -m pip install -r ./tortoise-tts/requirements.txt # install TorToiSe requirements +python3 -m pip install -e ./tortoise-tts/ # install TorToiSe +python3 -m pip install -r ./requirements.txt # install local requirements +# swap to ROCm version of BitsAndBytes +pip3 uninstall bitsandbytes +pip3 install git+https://github.com/broncotc/bitsandbytes-rocm + +deactivate \ No newline at end of file diff --git a/src/utils.py b/src/utils.py index 06cc7a9..e4d36be 100755 --- a/src/utils.py +++ b/src/utils.py @@ -477,7 +477,7 @@ def compute_latents(voice, voice_latents_chunks, progress=gr.Progress(track_tqdm # superfluous, but it cleans up some things class TrainingState(): - def __init__(self, config_path, keep_x_past_datasets=0): + def __init__(self, config_path, keep_x_past_datasets=0, start=True): self.cmd = ['train.bat', config_path] if os.name == "nt" else ['bash', './train.sh', config_path] # parse config to get its iteration @@ -527,8 +527,10 @@ class TrainingState(): self.losses = [] self.load_losses() - self.cleanup_old(keep=keep_x_past_datasets) - self.spawn_process() + if keep_x_past_datasets > 0: + self.cleanup_old(keep=keep_x_past_datasets) + if start: + self.spawn_process() def spawn_process(self): print("Spawning process: ", " ".join(self.cmd)) @@ -778,11 +780,19 @@ def get_training_losses(): return return pd.DataFrame(training_state.losses) -def update_training_dataplot(): +def update_training_dataplot(config_path=None): global training_state - if not training_state or not training_state.losses: - return - return gr.LinePlot.update(value=pd.DataFrame(training_state.losses)) + update = None + + if not training_state: + training_state = TrainingState(config_path=config_path, start=False) + update = gr.LinePlot.update(value=pd.DataFrame(training_state.losses)) + del training_state + training_state = None + else: + update = gr.LinePlot.update(value=pd.DataFrame(training_state.losses)) + + return update def reconnect_training(verbose=False, buffer_size=8, progress=gr.Progress(track_tqdm=True)): global training_state diff --git a/src/webui.py b/src/webui.py index 79153ef..a969033 100755 --- a/src/webui.py +++ b/src/webui.py @@ -527,17 +527,9 @@ def setup_gradio(): with gr.Row(): with gr.Column(): training_configs = gr.Dropdown(label="Training Configuration", choices=get_training_list()) - refresh_configs = gr.Button(value="Refresh Configurations") with gr.Row(): - start_training_button = gr.Button(value="Train") - stop_training_button = gr.Button(value="Stop") - reconnect_training_button = gr.Button(value="Reconnect") - with gr.Column(): - training_output = gr.TextArea(label="Console Output", interactive=False, max_lines=8) - verbose_training = gr.Checkbox(label="Verbose Console Output", value=True) - training_buffer_size = gr.Slider(label="Console Buffer Size", minimum=4, maximum=32, value=8) - training_keep_x_past_datasets = gr.Slider(label="Keep X Previous States", minimum=0, maximum=8, value=0, step=1) - + refresh_configs = gr.Button(value="Refresh Configurations") + training_loss_graph = gr.LinePlot(label="Training Metrics", x="step", y="value", @@ -545,8 +537,19 @@ def setup_gradio(): color="type", tooltip=['step', 'value', 'type'], width=600, - height=350 + height=350, ) + view_losses = gr.Button(value="View Losses") + + with gr.Column(): + training_output = gr.TextArea(label="Console Output", interactive=False, max_lines=8) + verbose_training = gr.Checkbox(label="Verbose Console Output", value=True) + training_buffer_size = gr.Slider(label="Console Buffer Size", minimum=4, maximum=32, value=8) + training_keep_x_past_datasets = gr.Slider(label="Keep X Previous States", minimum=0, maximum=8, value=0, step=1) + with gr.Row(): + start_training_button = gr.Button(value="Train") + stop_training_button = gr.Button(value="Stop") + reconnect_training_button = gr.Button(value="Reconnect") with gr.Tab("Settings"): with gr.Row(): exec_inputs = [] @@ -763,6 +766,17 @@ def setup_gradio(): ], show_progress=False, ) + + view_losses.click( + fn=update_training_dataplot, + inputs=[ + training_configs + ], + outputs=[ + training_loss_graph, + ], + ) + stop_training_button.click(stop_training, inputs=None, outputs=training_output #console_output diff --git a/start.bat b/start.bat index a3139a9..4ef2f49 100755 --- a/start.bat +++ b/start.bat @@ -1,5 +1,4 @@ call .\venv\Scripts\activate.bat set PATH=.\bin\;%PATH% python .\src\main.py %* -deactivate pause \ No newline at end of file diff --git a/train.bat b/train.bat index 55d682a..e745705 100755 --- a/train.bat +++ b/train.bat @@ -1,4 +1,4 @@ call .\venv\Scripts\activate.bat python ./src/train.py -opt "%1" -deactivate -pause \ No newline at end of file +pause +deactivate \ No newline at end of file diff --git a/update-force.bat b/update-force.bat index d5c5c56..4d77460 100755 --- a/update-force.bat +++ b/update-force.bat @@ -1,3 +1,15 @@ git fetch --all git reset --hard origin/master -call .\update.bat \ No newline at end of file +call .\update.bat + +python -m venv venv +call .\venv\Scripts\activate.bat + +python -m pip install --upgrade pip +python -m pip install -r .\dlas\requirements.txt +python -m pip install -r .\tortoise-tts\requirements.txt +python -m pip install -e .\tortoise-tts +python -m pip install -r .\requirements.txt + +pause +deactivate \ No newline at end of file diff --git a/update-force.sh b/update-force.sh index 1c94345..e8b1c2c 100755 --- a/update-force.sh +++ b/update-force.sh @@ -1,4 +1,17 @@ #!/bin/bash git fetch --all git reset --hard origin/master -./update.sh \ No newline at end of file + +./update.sh + +# force install requirements +python3 -m venv venv +source ./venv/bin/activate + +python3 -m pip install --upgrade pip +python3 -m pip install -r ./dlas/requirements.txt +python3 -m pip install -r ./tortoise-tts/requirements.txt +python3 -m pip install -e ./tortoise-tts +python3 -m pip install -r ./requirements.txt + +deactivate \ No newline at end of file diff --git a/update.bat b/update.bat index 843d5d8..9295634 100755 --- a/update.bat +++ b/update.bat @@ -1,14 +1,2 @@ git pull -git submodule update --remote - -python -m venv venv -call .\venv\Scripts\activate.bat - -python -m pip install --upgrade pip -python -m pip install -r .\dlas\requirements.txt -python -m pip install -r .\tortoise-tts\requirements.txt -python -m pip install -e .\tortoise-tts -python -m pip install -r .\requirements.txt - -deactivate -pause \ No newline at end of file +git submodule update --remote \ No newline at end of file diff --git a/update.sh b/update.sh index 584a8a4..c4a5f18 100755 --- a/update.sh +++ b/update.sh @@ -2,14 +2,4 @@ git pull git submodule update --remote -python3 -m venv venv -source ./venv/bin/activate - -python3 -m pip install --upgrade pip -python3 -m pip install -r ./dlas/requirements.txt -python3 -m pip install -r ./tortoise-tts/requirements.txt -python3 -m pip install -e ./tortoise-tts -python3 -m pip install -r ./requirements.txt - - deactivate \ No newline at end of file