From 83b512585410469ec573011f32d71ce9b0b092d4 Mon Sep 17 00:00:00 2001 From: mrq Date: Wed, 8 Mar 2023 03:29:12 +0000 Subject: [PATCH] fixed notebooks, provided paperspace notebook --- notebook.ipynb => notebook_colab.ipynb | 6 +- notebook_paperspace.ipynb | 132 +++++++++++++++++++++++++ src/utils.py | 23 +++-- 3 files changed, 146 insertions(+), 15 deletions(-) rename notebook.ipynb => notebook_colab.ipynb (97%) create mode 100755 notebook_paperspace.ipynb diff --git a/notebook.ipynb b/notebook_colab.ipynb similarity index 97% rename from notebook.ipynb rename to notebook_colab.ipynb index d41bc59..381648e 100755 --- a/notebook.ipynb +++ b/notebook_colab.ipynb @@ -51,10 +51,10 @@ "\n", "!python -m pip install --upgrade pip\n", "!pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu116\n", - "!python -m pip install -r ./dlas/requirements.txt\n", - "!python -m pip install -r ./tortoise-tts/requirements.txt\n", "!python -m pip install -r ./requirements.txt\n", + "!python -m pip install -r ./tortoise-tts/requirements.txt\n", "!python -m pip install -e ./tortoise-tts/\n", + "!python -m pip install -r ./dlas/requirements.txt\n", "\n", "!rm ./tortoise-tts/{main,webui}.py" ] @@ -152,7 +152,7 @@ "\n", "args = utils.setup_args()\n", "ui = webui.setup_gradio()\n", - "# Be very, very sure to check \"Defer TTS Load\" in Settings, then restart, before you start training\n", + "# Be very, very sure to check \"Do Not Load TTS On Startup\" in Settings after all the models download, then restart, before you start training\n", "# You'll crash the runtime if you don't\n", "if not args.defer_tts_load:\n", "\tutils.setup_tortoise()\n", diff --git a/notebook_paperspace.ipynb b/notebook_paperspace.ipynb new file mode 100755 index 0000000..7634b39 --- /dev/null +++ b/notebook_paperspace.ipynb @@ -0,0 +1,132 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "ni41hmE03DL6" + }, + "source": [ + "## Initialization" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "FtsMKKfH18iM" + }, + "source": [ + "!sudo apt update\n", + "!sudo apt-get install python3.9-venv -y\n", + "%cd /notebooks/\n", + "!git clone https://git.ecker.tech/mrq/ai-voice-cloning/\n", + "!ln -s ./ai-voice-cloning/models/ ./\n", + "%cd ai-voice-cloning\n", + "!./setup-cuda.sh\n", + "#!./update.sh" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IzrGt5IcHlAD" + }, + "source": [ + "# Update Repos" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "3DktoOXSHmtw" + }, + "source": [ + "# for my debugging purposes\n", + "%cd /notebooks/ai-voice-cloning/\n", + "!sudo apt update\n", + "!sudo apt-get install python3.9-venv -y\n", + "!mkdir -p ~/.cache\n", + "!ln -s /notebooks/ai-voice-cloning/models/voicefixer ~/.cache/.\n", + "!./update-force.sh\n", + "#!git pull\n", + "#!git submodule update --remote" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "o1gkfw3B3JSk" + }, + "source": [ + "## Running" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "c_EQZLTA19c7" + }, + "source": [ + "%cd /notebooks/ai-voice-cloning\n", + "\n", + "!export TORTOISE_MODELS_DIR=/notebooks/ai-voice-cloning/models/tortoise/\n", + "!export TRANSFORMERS_CACHE=/notebooks/ai-voice-cloning/models/transformers/\n", + "\n", + "!./start.sh --share" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2AnVQxEJx47p" + }, + "source": [ + "## Exporting" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "YOACiDCXx72G" + }, + "source": [ + "%cd /notebooks/ai-voice-cloning\n", + "!apt install -y p7zip-full\n", + "from datetime import datetime\n", + "timestamp = datetime.now().strftime('%m-%d-%Y_%H:%M:%S')\n", + "!mkdir -p \"../{timestamp}/results\"\n", + "!mv ./results/* \"../{timestamp}/results/.\"\n", + "!mv ./training/* \"../{timestamp}/training/.\"\n", + "!7z a -t7z -m0=lzma2 -mx=9 -mfb=64 -md=32m -ms=on \"../{timestamp}.7z\" \"../{timestamp}/\"\n", + "!ls ~/\n", + "!echo \"Finished zipping, archive is available at {timestamp}.7z\"" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "private_outputs": true, + "provenance": [] + }, + "gpuClass": "standard", + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/src/utils.py b/src/utils.py index 9752ca0..22ca7b4 100755 --- a/src/utils.py +++ b/src/utils.py @@ -697,23 +697,22 @@ class TrainingState(): logs = [logs[-1]] for log in logs: - try: ea = event_accumulator.EventAccumulator(log, size_guidance={event_accumulator.SCALARS: 0}) ea.Reload() for key in keys: - scalar = ea.Scalars(key) - for s in scalar: - if update and s.step <= self.last_info_check_at: - continue - highest_step = max( highest_step, s.step ) - self.statistics.append( { "step": s.step, "value": s.value, "type": key } ) + try: + scalar = ea.Scalars(key) + for s in scalar: + if update and s.step <= self.last_info_check_at: + continue + highest_step = max( highest_step, s.step ) + self.statistics.append( { "step": s.step, "value": s.value, "type": key } ) - if key == 'loss_gpt_total': - self.losses.append( { "step": s.step, "value": s.value, "type": key } ) - - except Exception as e: - pass + if key == 'loss_gpt_total': + self.losses.append( { "step": s.step, "value": s.value, "type": key } ) + except Exception as e: + pass else: logs = sorted([f'{self.dataset_dir}/{d}' for d in os.listdir(self.dataset_dir) if d[-4:] == ".log" ])