forked from mrq/ai-voice-cloning
Im not too sure if manually invoking gc actually closes all the open files from whisperx (or ROCm), but it seems to have gone away longside setting 'ulimit -Sn' to half the output of 'ulimit -Hn'
This commit is contained in:
parent
999878d9c6
commit
12c51b6057
|
@ -1054,6 +1054,7 @@ def prepare_dataset( files, outdir, language=None, skip_existings=False, progres
|
||||||
|
|
||||||
results = {}
|
results = {}
|
||||||
transcription = []
|
transcription = []
|
||||||
|
files = sorted(files)
|
||||||
|
|
||||||
previous_list = []
|
previous_list = []
|
||||||
if skip_existings and os.path.exists(f'{outdir}/train.txt'):
|
if skip_existings and os.path.exists(f'{outdir}/train.txt'):
|
||||||
|
@ -1103,6 +1104,8 @@ def prepare_dataset( files, outdir, language=None, skip_existings=False, progres
|
||||||
transcription.append(line)
|
transcription.append(line)
|
||||||
with open(f'{outdir}/train.txt', 'a', encoding="utf-8") as f:
|
with open(f'{outdir}/train.txt', 'a', encoding="utf-8") as f:
|
||||||
f.write(f'{line}\n')
|
f.write(f'{line}\n')
|
||||||
|
|
||||||
|
do_gc()
|
||||||
|
|
||||||
with open(f'{outdir}/whisper.json', 'w', encoding="utf-8") as f:
|
with open(f'{outdir}/whisper.json', 'w', encoding="utf-8") as f:
|
||||||
f.write(json.dumps(results, indent='\t'))
|
f.write(json.dumps(results, indent='\t'))
|
||||||
|
|
Loading…
Reference in New Issue
Block a user