tortoise-tts/tortoise/utils/text.py

import re


def split_and_recombine_text(text, desired_length=200, max_length=300):
    """Split text it into chunks of a desired length trying to keep sentences intact."""
    # normalize text, remove redundant whitespace and convert non-ascii quotes to ascii
    text = re.sub(r'\n\n+', '\n', text)
    text = re.sub(r'\s+', ' ', text)
    text = re.sub(r'[“”]', '"', text)

    rv = []
    in_quote = False
    current = ""
    split_pos = []
    pos = -1

    def seek(delta):
        nonlocal pos, in_quote, text
        is_neg = delta < 0
        for _ in range(abs(delta)):
            if is_neg:
                pos -= 1
            else:
                pos += 1
            if text[pos] == '"':
                in_quote = not in_quote
        return text[pos], text[pos+1] if pos < len(text)-1 else ""

    def commit():
        nonlocal rv, current, split_pos
        rv.append(current)
        current = ""
        split_pos = []

    while pos < len(text) - 1:
        c, next_c = seek(1)
        current += c
        # do we need to force a split?
        if len(current) >= max_length:
            if len(split_pos) > 0 and len(current) > (desired_length / 2):
                # we have at least one sentence and we are over half the desired length, seek back to the last split
                d = pos - split_pos[-1]
                seek(-d)
                current = current[:-d]
            else:
                # no full sentences, seek back until we are not in the middle of a word and split there
                while c not in '!?.\n ' and pos > 0 and len(current) > desired_length:
                    c, _ = seek(-1)
                    current = current[:-1]
            commit()
        # check for sentence boundaries
        elif not in_quote and (c in '!?\n' or (c == '.' and next_c in '\n ')):
            # seek forward if we have consecutive boundary markers but still within the max length
            while len(current) < max_length and next_c in '!?.':
                c, next_c = seek(1)
                current += c
            split_pos.append(pos)
            if len(current) >= desired_length:
                commit()
    rv.append(current)

    # clean up, remove lines with only whitespace or punctuation
    rv = [s.strip() for s in rv]
    rv = [s for s in rv if len(s) > 0 and not re.match(r'^[\s\.,;:!?]*$', s)]

    return rv


if __name__ == '__main__':
    import unittest

    class Test(unittest.TestCase):
        def test_split_and_recombine_text(self):
            text = """
            This is a sample sentence.
            This is another sample sentence.
            This is a longer sample sentence that should force a split inthemiddlebutinotinthislongword.
            "Don't split my quote... please"
            """
            self.assertEqual(split_and_recombine_text(text, desired_length=20, max_length=40),
                             ['This is a sample sentence.',
                              'This is another sample sentence.',
                              'This is a longer sample sentence that',
                              'should force a split',
                              'inthemiddlebutinotinthislongword.',
                              '"Don\'t split my quote... please"'])

        def test_split_and_recombine_text_2(self):
            text = """
            When you are really angry sometimes you use consecutive exclamation marks!!!!!! Is this a good thing to do?!?!?!
            I don't know but we should handle this situation..........................
            """
            self.assertEqual(split_and_recombine_text(text, desired_length=30, max_length=50),
                             ['When you are really angry sometimes you use',
                              'consecutive exclamation marks!!!!!!',
                              'Is this a good thing to do?!?!?!',
                              'I don\'t know but we should handle this situation.'])

    unittest.main()