From 8740cdefc61f524ef8019cd5fa78e8b71f5fa83e Mon Sep 17 00:00:00 2001 From: mrq Date: Wed, 11 Oct 2023 20:38:40 -0500 Subject: [PATCH] added initial support for languages (still testing, marked as model version 3), added experimental 'context extend by limiting the resp context' (untested) --- vall_e/__main__.py | 2 ++ vall_e/config.py | 11 ++++++-- vall_e/data.py | 58 ++++++++++++++++++++++++++++++++--------- vall_e/inference.py | 3 ++- vall_e/models/ar.py | 11 +++++--- vall_e/models/ar_nar.py | 19 ++++++++++++-- vall_e/models/base.py | 28 +++++++++++++------- vall_e/models/nar.py | 4 +-- 8 files changed, 104 insertions(+), 32 deletions(-) diff --git a/vall_e/__main__.py b/vall_e/__main__.py index b3a2875..9122cb9 100755 --- a/vall_e/__main__.py +++ b/vall_e/__main__.py @@ -17,6 +17,7 @@ def main(): parser.add_argument("--max-ar-steps", type=int, default=6 * 75) parser.add_argument("--max-nar-levels", type=int, default=7) + parser.add_argument("--max-ar-context", type=int, default=-1) parser.add_argument("--ar-temp", type=float, default=1.0) parser.add_argument("--nar-temp", type=float, default=1.0) @@ -46,6 +47,7 @@ def main(): out_path=args.out_path, input_prompt_length=args.input_prompt_length, max_ar_steps=args.max_ar_steps, max_nar_levels=args.max_nar_levels, + max_ar_context=args.max_ar_context, ar_temp=args.ar_temp, nar_temp=args.nar_temp, min_ar_temp=args.min_ar_temp, min_nar_temp=args.min_nar_temp, top_p=args.top_p, top_k=args.top_k, diff --git a/vall_e/config.py b/vall_e/config.py index 8fdd591..643ed6d 100755 --- a/vall_e/config.py +++ b/vall_e/config.py @@ -120,6 +120,9 @@ class Dataset: temp: list[Path] = field(default_factory=lambda: []) speaker_name_getter: str = "lambda p: f'{p.parts[-3]}_{p.parts[-2]}'" + speaker_group_getter: str = "lambda p: f'{p.parts[-3]}'" + + speaker_languages: dict = field(default_factory=lambda: {}) # dict where keys are the language codes and values are the speaker groups hdf5_name: str = "data.h5" use_hdf5: bool = False @@ -164,8 +167,8 @@ class Model: size: str | dict = "full" # preset string or explicitly defined dimensionality resp_levels: int = 1 # RVQ-bin levels this model targets for outputs prom_levels: int = 8 # RVQ-bin levels this model accepts as an input prompt - tasks: int = 0 # ["tts", "ns", "sr", "tse", "cse", "nse"] and leaves two more for anything else I want (like "svc") - langs: int = 0 # defined languages + tasks: int = 8 # ["tts", "ns", "sr", "tse", "cse", "nse"] and leaves two more for anything else I want (like "svc") + langs: int = 1 # defined languages arch_type: str = "retnet" # or "transformer"" training: bool = True # unneeded now interleave: bool = False # use an interleaved AR rather than a split AR + NAR (experimental, worse performance and results) @@ -518,6 +521,10 @@ class Config(_Config): def get_spkr(self): return eval(self.dataset.speaker_name_getter) + @cached_property + def get_spkr_group(self): + return eval(self.dataset.speaker_group_getter) + @cached_property def diskcache(self): if self.cfg_path is not None and self.dataset.cache: diff --git a/vall_e/data.py b/vall_e/data.py index 15833a6..4c5a0aa 100755 --- a/vall_e/data.py +++ b/vall_e/data.py @@ -33,20 +33,26 @@ def get_phone_symmap(): if cfg.dataset.use_hdf5 and 'symmap' in cfg.hdf5: return json.loads( cfg.hdf5['symmap'].asstr()[()] ) - symmap = {'': 1, '': 2, ' ': 3, '.': 4, ',': 5, '!': 6, '?': 7, 'p': 7, 'iː': 8, 'ɚ': 9, 'ˌ': 10, 'dˌ': 11, 'mˌ': 12, 'd': 13, 'ɹ': 14, 'tˈ': 15, 'pˌ': 16, 'uː': 17, 'l': 18, 'æ': 19, 'ɛ': 20, 'ɪ': 21, 'j': 22, 'ʊ': 23, 't': 24, 'n': 25, 'v': 26, 'a': 27, 'o': 28, 'ŋ': 29, 'w': 30, 'ʌ': 31, 'hˈ': 32, 'ɡˈ': 33, 'ə': 34, 'θˈ': 35, 'dˈ': 36, 'wˌ': 37, 'h': 38, 'z': 39, 'k': 40, 'ð': 41, 'ɡˌ': 42, 'ˈ': 43, 'fˈ': 44, 'i': 45, 's': 46, 'ʃ': 47, 'wˈ': 48, 'ðˈ': 49, 'ɹˈ': 50, 'lˈ': 51, 'ɡ': 52, 'oː': 53, 'mˈ': 54, 'e': 55, 'ɑː': 56, 'nˈ': 57, 'm': 58, 'θˌ': 59, 'sˈ': 60, 'f': 61, 'ɔː': 62, 'hˌ': 63, 'b': 64, 'jˈ': 65, 'ɐ': 66, 'ʒˈ': 67, 'θ': 68, 'bˈ': 69, 'ɾ': 70, 'ɜː': 71, 'ʌˈ': 72, 'ʃˌ': 73, 'bˌ': 74, 'kˈ': 75, 'ɔ': 76, 'zˈ': 77, 'ᵻ': 78, 'kˌ': 79, 'vˈ': 80, 'fˌ': 81, 'ʒ': 82, 'ʃˈ': 83, 'ɹˌ': 84, 'tˌ': 85, 'pˈ': 86, 'ðˌ': 87, 'sˌ': 88, 'nˌ': 89, 'lˌ': 90, '̩': 91, 'ʔ': 92, 'vˌ': 93, 'ɪˈ': 94, '"': 95, 'ɪˌ': 96, 'ʒˌ': 97, 'uːˌ': 98, 'ʊˈ': 99, 'jˌ': 100, 'uːˈ': 101, 'iːˈ': 102, 'zˌ': 103, '.ˈ': 104, '…': 105, 'ŋˌ': 106, 'ɐˌ': 107, '—ˈ': 108, 'iˌ': 109, 'iːˌ': 110, 'ɛː': 111, ')': 112, ')ˈ': 113, '(': 114, 'u': 115, '-': 116, 'ɖˈ': 117, 'iˈ': 118, 'ʰˈ': 119, 'ɟˈ': 120, '̃': 121, 'eː': 122, 'ɾˈ': 123, 'r': 124, 'ʰ': 125, '-ˌ': 126, 'ɫ': 127, 'q': 128, '—': 129, 'ʊˌ': 130, 'aː': 131, 'cˈ': 132, '…ˈ': 133, 'c': 134, 'ɳ': 135, 'ɐˈ': 136, 'x': 137, 'ʔˌ': 138, '.ˌ': 139, 'ɑ': 140, '?ˈ': 141, '̩ˈ': 142, '"ˈ': 143, ',ˈ': 144, 'ŋˈ': 145, 'əˌ': 146, '!ˈ': 147, '"ˌ': 148, '?ˌ': 149, ',ˌ': 150, '—ˌ': 151, '̩ˌ': 152, 'əˈ': 153, '!ˌ': 154, 'ɬ': 155, 'ʲ': 156, '¡': 157, 'ɯ': 158, 'qˌ': 159, 'ʑ': 160, 'ʑˈ': 161, '¿': 162, 'ɑːˈ': 163, 'iːː': 164, 'ɛˈ': 165, '¡ˈ': 166, 'æˈ': 167, 'ç': 168, 'ɾˌ': 169, 'ᵻˈ': 170, 'xˈ': 171, 'ɔːˈ': 172, ';': 173, 'ɬˌ': 174, ':': 175, 'ʔˈ': 176, 'ɑːˌ': 177, 'ɬˈ': 178, '”': 179, '“': 180, '“ˈ': 181, '“ˌ': 182, ';ˈ': 183, ';ˌ': 184, ':ˈ': 185} + symmap = {'': 1, '': 2, ' ': 3, '.': 4, ',': 5, '!': 6, '?': 7, 'p': 7, 'iː': 8, 'ɚ': 9, 'ˌ': 10, 'dˌ': 11, 'mˌ': 12, 'd': 13, 'ɹ': 14, 'tˈ': 15, 'pˌ': 16, 'uː': 17, 'l': 18, 'æ': 19, 'ɛ': 20, 'ɪ': 21, 'j': 22, 'ʊ': 23, 't': 24, 'n': 25, 'v': 26, 'a': 27, 'o': 28, 'ŋ': 29, 'w': 30, 'ʌ': 31, 'hˈ': 32, 'ɡˈ': 33, 'ə': 34, 'θˈ': 35, 'dˈ': 36, 'wˌ': 37, 'h': 38, 'z': 39, 'k': 40, 'ð': 41, 'ɡˌ': 42, 'ˈ': 43, 'fˈ': 44, 'i': 45, 's': 46, 'ʃ': 47, 'wˈ': 48, 'ðˈ': 49, 'ɹˈ': 50, 'lˈ': 51, 'ɡ': 52, 'oː': 53, 'mˈ': 54, 'e': 55, 'ɑː': 56, 'nˈ': 57, 'm': 58, 'θˌ': 59, 'sˈ': 60, 'f': 61, 'ɔː': 62, 'hˌ': 63, 'b': 64, 'jˈ': 65, 'ɐ': 66, 'ʒˈ': 67, 'θ': 68, 'bˈ': 69, 'ɾ': 70, 'ɜː': 71, 'ʌˈ': 72, 'ʃˌ': 73, 'bˌ': 74, 'kˈ': 75, 'ɔ': 76, 'zˈ': 77, 'ᵻ': 78, 'kˌ': 79, 'vˈ': 80, 'fˌ': 81, 'ʒ': 82, 'ʃˈ': 83, 'ɹˌ': 84, 'tˌ': 85, 'pˈ': 86, 'ðˌ': 87, 'sˌ': 88, 'nˌ': 89, 'lˌ': 90, '̩': 91, 'ʔ': 92, 'vˌ': 93, 'ɪˈ': 94, '"': 95, 'ɪˌ': 96, 'ʒˌ': 97, 'uːˌ': 98, 'ʊˈ': 99, 'jˌ': 100, 'uːˈ': 101, 'iːˈ': 102, 'zˌ': 103, '.ˈ': 104, '…': 105, 'ŋˌ': 106, 'ɐˌ': 107, '—ˈ': 108, 'iˌ': 109, 'iːˌ': 110, 'ɛː': 111, ')': 112, ')ˈ': 113, '(': 114, 'u': 115, '-': 116, 'ɖˈ': 117, 'iˈ': 118, 'ʰˈ': 119, 'ɟˈ': 120, '̃': 121, 'eː': 122, 'ɾˈ': 123, 'r': 124, 'ʰ': 125, '-ˌ': 126, 'ɫ': 127, 'q': 128, '—': 129, 'ʊˌ': 130, 'aː': 131, 'cˈ': 132, '…ˈ': 133, 'c': 134, 'ɳ': 135, 'ɐˈ': 136, 'x': 137, 'ʔˌ': 138, '.ˌ': 139, 'ɑ': 140, '?ˈ': 141, '̩ˈ': 142, '"ˈ': 143, ',ˈ': 144, 'ŋˈ': 145, 'əˌ': 146, '!ˈ': 147, '"ˌ': 148, '?ˌ': 149, ',ˌ': 150, '—ˌ': 151, '̩ˌ': 152, 'əˈ': 153, '!ˌ': 154, 'ɬ': 155, 'ʲ': 156, '¡': 157, 'ɯ': 158, 'qˌ': 159, 'ʑ': 160, 'ʑˈ': 161, '¿': 162, 'ɑːˈ': 163, 'iːː': 164, 'ɛˈ': 165, '¡ˈ': 166, 'æˈ': 167, 'ç': 168, 'ɾˌ': 169, 'ᵻˈ': 170, 'xˈ': 171, 'ɔːˈ': 172, ';': 173, 'ɬˌ': 174, ':': 175, 'ʔˈ': 176, 'ɑːˌ': 177, 'ɬˈ': 178, '”': 179, '“': 180, '“ˈ': 181, '“ˌ': 182, ';ˈ': 183, ';ˌ': 184, ':ˈ': 185, '1': 186, 'rˈ': 187, 'qˈ': 188, 'ᵻˌ': 189, 'ä': 190, '̞ˌ': 191, '̞': 192, 'ũˌ': 193, 'ʑˌ': 194, 'ᵝ': 195, 'ɽ': 196, 'ʲˌ': 197, 'ᵝˌ': 198, 'ũ': 199, 'ũˈ': 200, 'äˌ': 201, 'ɕ': 202, 'ɕˌ': 203, 'ɽˌ': 204, 'çˌ': 205, '…ˌ': 206, '̞ˈ': 207, 'äˈ': 208, 'ɽˈ': 209, 'ɸˌ': 210, 'ɴ': 211, 'ɸˈ': 212, 'ɕˈ': 213, 'ɸ': 214, 'ᵝˈ': 215, 'ʲˈ': 216, 'ĩ': 217, 'çˈ': 218, 'ĩˌ': 219, 'oˌ': 220, 'eˈ': 221, 'ʍ': 222, 'eˌ': 223, 'uˌ': 224, 'ʍˌ': 225, 'uˈ': 226, 'oˈ': 227, 'aˈ': 228} + return symmap + +def get_lang_symmap(): + symmap = { + "en": 0, + "ja": 1, + } return symmap def get_task_symmap(): - start = 1024 symmap = { - "": -100, - "": start + 0, - "": start + 1, - "": start + 2, - "": start + 3, - "": start + 4, - "": start + 5, - "": start + 6, + "": 0, + "": 1, + "": 2, + "": 3, + "": 4, + "": 5, + "": 6, + "": 7, } return symmap @@ -105,7 +111,9 @@ def _get_hdf5_path(path): path = str(path) if path[:2] != "./": path = f'./{path}' - return path.replace(cfg.cfg_path, "") + + res = path.replace(cfg.cfg_path, "") + return res def _get_hdf5_paths( data_dir, type="training", validate=False ): data_dir = str(data_dir) @@ -206,6 +214,7 @@ class Dataset(_Dataset): self.phone_symmap = phone_symmap or self._get_phone_symmap() self.spkr_symmap = self._get_spkr_symmap() + self.lang_symmap = self._get_lang_symmap() self.task_symmap = self._get_task_symmap() # assert len(self.phone_symmap) < 256, "Unique token count should be [0,255] to fit within uint8" @@ -227,6 +236,21 @@ class Dataset(_Dataset): res = cfg.get_spkr(path) return res + def get_speaker_group(self, path): + if isinstance(path, str): + path = Path(path) + res = cfg.get_spkr_group(path) + return res + + def get_language(self, speaker_group): + lang = "en" + for k, v in cfg.dataset.speaker_languages.items(): + if speaker_group in v: + lang = k + break + + return lang + @cached_property def spkrs(self): return sorted({self.get_speaker(path) for path in self.paths}) @@ -257,13 +281,18 @@ class Dataset(_Dataset): def _get_spkr_symmap(self): return {s: i for i, s in enumerate(self.spkrs)} + def _get_lang_symmap(self): + return get_lang_symmap() + def _get_task_symmap(self): return get_task_symmap() + """ def get_task_token( self, token, levels=cfg.models.max_levels ): if not hasattr(self, "task_symmap"): self.task_symmap = self._get_task_symmap() return torch.Tensor([[ self.task_symmap[f'<{token}>'] for _ in range(levels) ]]).to(dtype=torch.int16) + """ def sample_noise(self): path = random.choice(self.noise_paths) @@ -340,6 +369,7 @@ class Dataset(_Dataset): if cfg.dataset.use_hdf5: key = _get_hdf5_path(path) + text = cfg.hdf5[key]["text"][:] resps = cfg.hdf5[key]["audio"][:, :] @@ -351,6 +381,9 @@ class Dataset(_Dataset): text = torch.tensor([*map(self.phone_symmap.get, _get_phones(path))]).to(self.text_dtype) resps = _load_quants(path) + spkr_group = self.get_speaker_group(path) + lang = self.lang_symmap[ self.get_language(spkr_group) ] + # append additional prompts in an attempt to artifically increase lengths / offer new data if cfg.experimental and cfg.dataset.max_resps > 1 and random.random() < cfg.dataset.p_resp_append: choices = [*(set(self.paths_by_spkr_name[spkr_name]) - {path})] @@ -565,6 +598,7 @@ class Dataset(_Dataset): spkr_name=spkr_name, spkr_id=spkr_id, task=task, + lang=lang, text=text, proms=proms, resps=resps, @@ -799,8 +833,8 @@ def create_dataset_hdf5( skip_existing=True ): # write symmap if "symmap" in hf: del hf['symmap'] - hf.create_dataset('symmap', data=json.dumps(symmap)) + hf.create_dataset('symmap', data=json.dumps(symmap)) hf.close() if __name__ == "__main__": diff --git a/vall_e/inference.py b/vall_e/inference.py index 0b9e5b9..e39d13e 100755 --- a/vall_e/inference.py +++ b/vall_e/inference.py @@ -149,6 +149,7 @@ class TTS(): text, references, max_ar_steps=6 * 75, + max_ar_context=-1, max_nar_levels=7, input_prompt_length=0.0, ar_temp=0.95, @@ -176,7 +177,7 @@ class TTS(): with torch.autocast("cuda", dtype=self.dtype, enabled=self.amp): resps_list = self.ar( - text_list=[phns], proms_list=[prom], max_steps=max_ar_steps, + text_list=[phns], proms_list=[prom], max_steps=max_ar_steps, max_resp_context=max_ar_context, sampling_temperature=ar_temp, sampling_min_temperature=min_ar_temp, sampling_top_p=top_p, sampling_top_k=top_k, diff --git a/vall_e/models/ar.py b/vall_e/models/ar.py index c83d27e..f3716eb 100755 --- a/vall_e/models/ar.py +++ b/vall_e/models/ar.py @@ -39,11 +39,11 @@ class AR(Base): @property def n_tasks(self) -> int: - return cfg.models.tasks + return cfg.models.ar.tasks @property def n_langs(self) -> int: - return cfg.models.langs + return cfg.models.ar.langs @property def recurrent_chunk_size(self) -> int: @@ -103,6 +103,7 @@ class AR(Base): proms_list: list[Tensor], resps_list: list[Tensor] | None = None, max_steps: int = 1000, + max_resp_context: int = -1, sampling_temperature: float = 1.0, sampling_min_temperature: float = -1.0, @@ -149,7 +150,11 @@ class AR(Base): # get next in sequence for n in trange(max_steps // max(1, self.recurrent_chunk_size)): - resps_list = self._unsqueeze_list(sequence_list) + if max_resp_context > 0: + resps_list = self._unsqueeze_list([ sequence[-max_resp_context:] for sequence in sequence_list ] ) + else: + resps_list = self._unsqueeze_list(sequence_list) + logits = super().forward( text_list=text_list, proms_list=proms_list, diff --git a/vall_e/models/ar_nar.py b/vall_e/models/ar_nar.py index 6ae02f8..4bd2679 100644 --- a/vall_e/models/ar_nar.py +++ b/vall_e/models/ar_nar.py @@ -43,7 +43,11 @@ class AR_NAR(Base): @property def n_tasks(self) -> int: - return cfg.models.tasks + return cfg.models.ar_nar.tasks + + @property + def n_langs(self) -> int: + return cfg.models.ar_nar.langs @property def recurrent_chunk_size(self) -> int: @@ -86,8 +90,13 @@ class AR_NAR(Base): text_list: list[Tensor], proms_list: list[Tensor], resps_list: list[Tensor] | None = None, + + lang_list: list[Tensor] | None = None, + max_steps: int = 1000, max_levels: int = 7, + max_resp_context: int = -1, + sampling_temperature: float = 1.0, sampling_min_temperature: float = -1.0, sampling_top_k: int = -100, @@ -184,7 +193,13 @@ class AR_NAR(Base): # get next in sequence for n in trange(max_steps // max(1, self.recurrent_chunk_size)): - resps_list = self._unsqueeze_list(sequence_list) + # experimental rolling response to avoid too-long perplexity hits despite RetNet allegedly fixing this. + # UNTESTED. In theory it would be better to also adjust the text, but there's no way of correlating text to segment of audio without something like wav2vec2 + if max_resp_context > 0: + resps_list = self._unsqueeze_list([ sequence[-max_resp_context:] for sequence in sequence_list ] ) + else: + resps_list = self._unsqueeze_list(sequence_list) + logits = super().forward( text_list=text_list, proms_list=proms_list, diff --git a/vall_e/models/base.py b/vall_e/models/base.py index 9f6d987..f153952 100755 --- a/vall_e/models/base.py +++ b/vall_e/models/base.py @@ -191,7 +191,7 @@ class Base(nn.Module): cat = torch.cat else: cat = partial(_join, sep=sep) - return [*map(cat, zip(*l))] + return [*map(cat, zip([x for x in l if x is not None]))] def __init__( self, @@ -229,8 +229,9 @@ class Base(nn.Module): # [1025] + [1024] * 8 self.resps_emb = AudioEmbedding([n_resp_tokens] + [n_resp_tokens - 1] * (self.n_resp_levels - 1), d_model) - # self.langs_emb = Embedding(self.n_langs, d_model) - # self.tasks_emb = Embedding(self.n_tasks, d_model) + if self.version >= 3: + self.langs_emb = Embedding(self.n_langs, d_model) + self.tasks_emb = Embedding(self.n_tasks, d_model) self.sep = nn.Parameter(torch.randn(d_model)) @@ -291,25 +292,27 @@ class Base(nn.Module): proms_list: list[Tensor], resps_list: list[Tensor], targ_list: list[Tensor] | None = None, - - #langs_list: list[Tensor] | None = None, - #tasks_list: list[Tensor] | None = None, + + lang_list: list[Tensor] | None = None, quant_levels: Tensor | None = None, state: dict | None = None, ): + batch_size = len(text_list) + + if self.langs_emb is None: + langs_list = None + x_list = self._samplewise_merge_tensors( self.text_emb(text_list), - #self.langs_emb(langs_list), + self.langs_emb(lang_list) if lang_list is not None else None, self.proms_emb(proms_list), - #self.tasks_emb(tasks_list), self.resps_emb(resps_list, quant_levels), sep=self.sep, ) x, m = list_to_tensor(x_list) - batch_size = len(text_list) device = x.device if state is not None and self.arch_type == "retnet": @@ -349,7 +352,12 @@ class Base(nn.Module): # create a tensor sequence with one RVQ-bin of the input prompt, but with `ignore_index`, as the prompt is not neeeded for computing the loss against prom_list = [ torch.full_like(t[..., 0], self.ignore_index) for t in proms_list ] # remake input sequence - text_prom_list = self._samplewise_merge_tensors( text_list, prom_list, sep=ignore_sep ) + text_prom_list = self._samplewise_merge_tensors( + text_list, + lang_list, + prom_list, + sep=ignore_sep + ) # process each batch for i in range(len(text_prom_list)): diff --git a/vall_e/models/nar.py b/vall_e/models/nar.py index 1366f32..91d8d34 100755 --- a/vall_e/models/nar.py +++ b/vall_e/models/nar.py @@ -37,11 +37,11 @@ class NAR(Base): @property def n_tasks(self) -> int: - return cfg.models.tasks + return cfg.models.nar.tasks @property def n_langs(self) -> int: - return cfg.models.langs + return cfg.models.nar.langs @property def version(self) -> int: