From b64948d966373496e7cac5b39d78cb00ec94a700 Mon Sep 17 00:00:00 2001 From: mrq Date: Thu, 9 Mar 2023 04:03:57 +0000 Subject: [PATCH] while I'm breaking things, migrating dependencies to modules folder for tidiness --- .gitmodules | 4 ++-- dlas => modules/dlas | 0 tortoise-tts => modules/tortoise-tts | 0 setup-cuda.bat | 14 ++++++++------ setup-cuda.sh | 7 ++++--- setup-directml.bat | 8 +++++--- setup-rocm-bnb.sh | 9 ++++++--- setup-rocm.sh | 11 ++++++----- src/train.py | 4 ++-- update-force.bat | 6 +++--- update-force.sh | 6 +++--- 11 files changed, 39 insertions(+), 30 deletions(-) rename dlas => modules/dlas (100%) rename tortoise-tts => modules/tortoise-tts (100%) diff --git a/.gitmodules b/.gitmodules index 957430f..240179d 100755 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "tortoise-tts"] - path = tortoise-tts + path = modules/tortoise-tts url = https://git.ecker.tech/mrq/tortoise-tts [submodule "dlas"] - path = dlas + path = modules/dlas url = https://git.ecker.tech/mrq/DL-Art-School diff --git a/dlas b/modules/dlas similarity index 100% rename from dlas rename to modules/dlas diff --git a/tortoise-tts b/modules/tortoise-tts similarity index 100% rename from tortoise-tts rename to modules/tortoise-tts diff --git a/setup-cuda.bat b/setup-cuda.bat index efd7262..14b6d5f 100755 --- a/setup-cuda.bat +++ b/setup-cuda.bat @@ -6,13 +6,15 @@ call .\venv\Scripts\activate.bat python -m pip install --upgrade pip python -m pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu117 python -m pip install -r .\requirements.txt -python -m pip install -r .\tortoise-tts\requirements.txt -python -m pip install -e .\tortoise-tts\ -python -m pip install -r .\dlas\requirements.txt +python -m pip install -r .\modules\tortoise-tts\requirements.txt +python -m pip install -e .\modules\tortoise-tts\ +python -m pip install -r .\modules\dlas\requirements.txt -xcopy .\dlas\bitsandbytes_windows\* .\venv\Lib\site-packages\bitsandbytes\. /Y -xcopy .\dlas\bitsandbytes_windows\cuda_setup\* .\venv\Lib\site-packages\bitsandbytes\cuda_setup\. /Y -xcopy .\dlas\bitsandbytes_windows\nn\* .\venv\Lib\site-packages\bitsandbytes\nn\. /Y +xcopy .\modules\dlas\bitsandbytes_windows\* .\venv\Lib\site-packages\bitsandbytes\. /Y +xcopy .\modules\dlas\bitsandbytes_windows\cuda_setup\* .\venv\Lib\site-packages\bitsandbytes\cuda_setup\. /Y +xcopy .\modules\dlas\bitsandbytes_windows\nn\* .\venv\Lib\site-packages\bitsandbytes\nn\. /Y + +del *.sh pause deactivate \ No newline at end of file diff --git a/setup-cuda.sh b/setup-cuda.sh index bd0628e..c494375 100755 --- a/setup-cuda.sh +++ b/setup-cuda.sh @@ -10,8 +10,9 @@ python3 -m pip install --upgrade pip # just to be safe pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu117 # install requirements python3 -m pip install -r ./requirements.txt # install local requirements -python3 -m pip install -r ./tortoise-tts/requirements.txt # install TorToiSe requirements -python3 -m pip install -e ./tortoise-tts/ # install TorToiSe -python3 -m pip install -r ./dlas/requirements.txt # instal DLAS requirements, last, because whisperx will break a dependency here +python3 -m pip install -r ./modules/tortoise-tts/requirements.txt # install TorToiSe requirements +python3 -m pip install -e ./modules/tortoise-tts/ # install TorToiSe +python3 -m pip install -r ./modules/dlas/requirements.txt # instal DLAS requirements, last, because whisperx will break a dependency here +rm *.bat deactivate \ No newline at end of file diff --git a/setup-directml.bat b/setup-directml.bat index 9a11229..35f8008 100755 --- a/setup-directml.bat +++ b/setup-directml.bat @@ -6,9 +6,11 @@ call .\venv\Scripts\activate.bat python -m pip install --upgrade pip python -m pip install torch torchvision torchaudio torch-directml python -m pip install -r .\requirements.txt -python -m pip install -r .\tortoise-tts\requirements.txt -python -m pip install -e .\tortoise-tts\ -python -m pip install -r .\dlas\requirements.txt +python -m pip install -r .\modules\tortoise-tts\requirements.txt +python -m pip install -e .\modules\tortoise-tts\ +python -m pip install -r .\modules\dlas\requirements.txt + +del *.sh pause deactivate \ No newline at end of file diff --git a/setup-rocm-bnb.sh b/setup-rocm-bnb.sh index eff17f2..f94072d 100755 --- a/setup-rocm-bnb.sh +++ b/setup-rocm-bnb.sh @@ -1,8 +1,11 @@ #!/bin/bash source ./venv/bin/activate -git clone https://git.ecker.tech/mrq/bitsandbytes-rocm -cd bitsandbytes-rocm +# swap to ROCm version of BitsAndBytes +pip3 uninstall -y bitsandbytes + +git clone https://git.ecker.tech/mrq/bitsandbytes-rocm modules/bitsandbytes-rocm +cd modules/bitsandbytes-rocm make hip CUDA_VERSION=gfx1030 python setup.py install # assumes you're using a 6XXX series card python3 -m bitsandbytes # to validate it works -cd .. \ No newline at end of file +cd ../.. \ No newline at end of file diff --git a/setup-rocm.sh b/setup-rocm.sh index c5d67b8..9e729d9 100755 --- a/setup-rocm.sh +++ b/setup-rocm.sh @@ -10,10 +10,11 @@ python3 -m pip install --upgrade pip # just to be safe pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/rocm5.1.1 # 5.2 does not work for me desu # install requirements python3 -m pip install -r ./requirements.txt # install local requirements -python3 -m pip install -r ./tortoise-tts/requirements.txt # install TorToiSe requirements -python3 -m pip install -e ./tortoise-tts/ # install TorToiSe -python3 -m pip install -r ./dlas/requirements.txt # instal DLAS requirements -# swap to ROCm version of BitsAndBytes -pip3 uninstall -y bitsandbytes +python3 -m pip install -r ./modules/tortoise-tts/requirements.txt # install TorToiSe requirements +python3 -m pip install -e ./modules/tortoise-tts/ # install TorToiSe +python3 -m pip install -r ./modules/dlas/requirements.txt # instal DLAS requirements +rm *.bat + ./setup-rocm-bnb.sh + deactivate \ No newline at end of file diff --git a/src/train.py b/src/train.py index 78f4ff3..72b3f38 100755 --- a/src/train.py +++ b/src/train.py @@ -33,10 +33,10 @@ if __name__ == "__main__": # this is some massive kludge that only works if it's called from a shell and not an import/PIP package # it's smart-yet-irritating module-model loader breaks when trying to load something specifically when not from a shell -sys.path.insert(0, './dlas/codes/') +sys.path.insert(0, './modules/dlas/codes/') # this is also because DLAS is not written as a package in mind # it'll gripe when it wants to import from train.py -sys.path.insert(0, './dlas/') +sys.path.insert(0, './modules/dlas/') # for PIP, replace it with: # sys.path.insert(0, os.path.dirname(os.path.realpath(dlas.__file__))) diff --git a/update-force.bat b/update-force.bat index 75cde86..fcbe106 100755 --- a/update-force.bat +++ b/update-force.bat @@ -7,9 +7,9 @@ call .\venv\Scripts\activate.bat python -m pip install --upgrade pip python -m pip install -U -r .\requirements.txt -python -m pip install -U -r .\tortoise-tts\requirements.txt -python -m pip install -U -e .\tortoise-tts -python -m pip install -U -r .\dlas\requirements.txt +python -m pip install -U -r .\modules\tortoise-tts\requirements.txt +python -m pip install -U -e .\modules\tortoise-tts +python -m pip install -U -r .\modules\dlas\requirements.txt pause deactivate \ No newline at end of file diff --git a/update-force.sh b/update-force.sh index 5999566..5e8c808 100755 --- a/update-force.sh +++ b/update-force.sh @@ -10,8 +10,8 @@ source ./venv/bin/activate python3 -m pip install --upgrade pip python3 -m pip install -r ./requirements.txt -python3 -m pip install -r ./tortoise-tts/requirements.txt -python3 -m pip install -e ./tortoise-tts -python3 -m pip install -r ./dlas/requirements.txt +python3 -m pip install -r ./modules/tortoise-tts/requirements.txt +python3 -m pip install -e ./modules/tortoise-tts +python3 -m pip install -r ./modules/dlas/requirements.txt deactivate \ No newline at end of file