backported caching of phonemizer backend from mrq/vall-e

This commit is contained in:
mrq 2023-04-28 15:31:45 +00:00
parent c5e9b407fa
commit 99387920e1

View File

@ -2277,11 +2277,38 @@ def create_dataset_json( path ):
with open(path.replace(".txt", ".json"), 'w', encoding='utf-8') as f:
f.write(json.dumps(data, indent="\t"))
cached_backends = {}
def phonemizer( text, language="en-us" ):
from phonemizer import phonemize
from phonemizer.backend import BACKENDS
def _get_backend( language="en-us", backend="espeak" ):
key = f'{language}_{backend}'
if key in cached_backends:
return cached_backends[key]
if backend == 'espeak':
phonemizer = BACKENDS[backend]( language, preserve_punctuation=True, with_stress=True)
elif backend == 'espeak-mbrola':
phonemizer = BACKENDS[backend]( language )
else:
phonemizer = BACKENDS[backend]( language, preserve_punctuation=True )
cached_backends[key] = phonemizer
return phonemizer
if language == "en":
language = "en-us"
return phonemize( text, language=language, strip=True, preserve_punctuation=True, with_stress=True, backend=args.phonemizer_backend )
backend = _get_backend(language=language, backend=args.phonemizer_backend)
if backend is not None:
tokens = backend.phonemize( text, strip=True )
else:
tokens = phonemize( text, language=language, strip=True, preserve_punctuation=True, with_stress=True )
return tokens[0] if len(tokens) == 0 else tokens
tokenized = " ".join( tokens )
def should_phonemize():
should = args.tokenizer_json is not None and args.tokenizer_json[-8:] == "ipa.json"