Delete text

text/__init__.py

@@ -1,30 +0,0 @@
-from text.symbols import *
-
-_symbol_to_id = {s: i for i, s in enumerate(symbols)}
-
-
-def cleaned_text_to_sequence(cleaned_text, tones, language):
-    """Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
-    Args:
-      text: string to convert to a sequence
-    Returns:
-      List of integers corresponding to the symbols in the text
-    """
-    phones = [_symbol_to_id[symbol] for symbol in cleaned_text]
-    tone_start = language_tone_start_map[language]
-    tones = [i + tone_start for i in tones]
-    lang_id = language_id_map[language]
-    lang_ids = [lang_id for i in phones]
-    return phones, tones, lang_ids
-
-
-def get_bert(norm_text, word2ph, language, device, style_text=None, style_weight=0.7):
-    from .chinese_bert import get_bert_feature as zh_bert
-    from .english_bert_mock import get_bert_feature as en_bert
-    from .japanese_bert import get_bert_feature as jp_bert
-
-    lang_bert_func_map = {"ZH": zh_bert, "EN": en_bert, "JP": jp_bert}
-    bert = lang_bert_func_map[language](
-        norm_text, word2ph, device, style_text, style_weight
-    )
-    return bert

text/chinese.py

@@ -1,199 +0,0 @@
-import os
-import re
-
-import cn2an
-from pypinyin import lazy_pinyin, Style
-
-from text.symbols import punctuation
-from text.tone_sandhi import ToneSandhi
-
-current_file_path = os.path.dirname(__file__)
-pinyin_to_symbol_map = {
-    line.split("\t")[0]: line.strip().split("\t")[1]
-    for line in open(os.path.join(current_file_path, "opencpop-strict.txt")).readlines()
-}
-
-import jieba.posseg as psg
-
-
-rep_map = {
-    "：": ",",
-    "；": ",",
-    "，": ",",
-    "。": ".",
-    "！": "!",
-    "？": "?",
-    "\n": ".",
-    "·": ",",
-    "、": ",",
-    "...": "…",
-    "$": ".",
-    "“": "'",
-    "”": "'",
-    '"': "'",
-    "‘": "'",
-    "’": "'",
-    "（": "'",
-    "）": "'",
-    "(": "'",
-    ")": "'",
-    "《": "'",
-    "》": "'",
-    "【": "'",
-    "】": "'",
-    "[": "'",
-    "]": "'",
-    "—": "-",
-    "～": "-",
-    "~": "-",
-    "「": "'",
-    "」": "'",
-}
-
-tone_modifier = ToneSandhi()
-
-
-def replace_punctuation(text):
-    text = text.replace("嗯", "恩").replace("呣", "母")
-    pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
-
-    replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
-
-    replaced_text = re.sub(
-        r"[^\u4e00-\u9fa5" + "".join(punctuation) + r"]+", "", replaced_text
-    )
-
-    return replaced_text
-
-
-def g2p(text):
-    pattern = r"(?<=[{0}])\s*".format("".join(punctuation))
-    sentences = [i for i in re.split(pattern, text) if i.strip() != ""]
-    phones, tones, word2ph = _g2p(sentences)
-    assert sum(word2ph) == len(phones)
-    assert len(word2ph) == len(text)  # Sometimes it will crash,you can add a try-catch.
-    phones = ["_"] + phones + ["_"]
-    tones = [0] + tones + [0]
-    word2ph = [1] + word2ph + [1]
-    return phones, tones, word2ph
-
-
-def _get_initials_finals(word):
-    initials = []
-    finals = []
-    orig_initials = lazy_pinyin(word, neutral_tone_with_five=True, style=Style.INITIALS)
-    orig_finals = lazy_pinyin(
-        word, neutral_tone_with_five=True, style=Style.FINALS_TONE3
-    )
-    for c, v in zip(orig_initials, orig_finals):
-        initials.append(c)
-        finals.append(v)
-    return initials, finals
-
-
-def _g2p(segments):
-    phones_list = []
-    tones_list = []
-    word2ph = []
-    for seg in segments:
-        # Replace all English words in the sentence
-        seg = re.sub("[a-zA-Z]+", "", seg)
-        seg_cut = psg.lcut(seg)
-        initials = []
-        finals = []
-        seg_cut = tone_modifier.pre_merge_for_modify(seg_cut)
-        for word, pos in seg_cut:
-            if pos == "eng":
-                continue
-            sub_initials, sub_finals = _get_initials_finals(word)
-            sub_finals = tone_modifier.modified_tone(word, pos, sub_finals)
-            initials.append(sub_initials)
-            finals.append(sub_finals)
-
-            # assert len(sub_initials) == len(sub_finals) == len(word)
-        initials = sum(initials, [])
-        finals = sum(finals, [])
-        #
-        for c, v in zip(initials, finals):
-            raw_pinyin = c + v
-            # NOTE: post process for pypinyin outputs
-            # we discriminate i, ii and iii
-            if c == v:
-                assert c in punctuation
-                phone = [c]
-                tone = "0"
-                word2ph.append(1)
-            else:
-                v_without_tone = v[:-1]
-                tone = v[-1]
-
-                pinyin = c + v_without_tone
-                assert tone in "12345"
-
-                if c:
-                    # 多音节
-                    v_rep_map = {
-                        "uei": "ui",
-                        "iou": "iu",
-                        "uen": "un",
-                    }
-                    if v_without_tone in v_rep_map.keys():
-                        pinyin = c + v_rep_map[v_without_tone]
-                else:
-                    # 单音节
-                    pinyin_rep_map = {
-                        "ing": "ying",
-                        "i": "yi",
-                        "in": "yin",
-                        "u": "wu",
-                    }
-                    if pinyin in pinyin_rep_map.keys():
-                        pinyin = pinyin_rep_map[pinyin]
-                    else:
-                        single_rep_map = {
-                            "v": "yu",
-                            "e": "e",
-                            "i": "y",
-                            "u": "w",
-                        }
-                        if pinyin[0] in single_rep_map.keys():
-                            pinyin = single_rep_map[pinyin[0]] + pinyin[1:]
-
-                assert pinyin in pinyin_to_symbol_map.keys(), (pinyin, seg, raw_pinyin)
-                phone = pinyin_to_symbol_map[pinyin].split(" ")
-                word2ph.append(len(phone))
-
-            phones_list += phone
-            tones_list += [int(tone)] * len(phone)
-    return phones_list, tones_list, word2ph
-
-
-def text_normalize(text):
-    numbers = re.findall(r"\d+(?:\.?\d+)?", text)
-    for number in numbers:
-        text = text.replace(number, cn2an.an2cn(number), 1)
-    text = replace_punctuation(text)
-    return text
-
-
-def get_bert_feature(text, word2ph):
-    from text import chinese_bert
-
-    return chinese_bert.get_bert_feature(text, word2ph)
-
-
-if __name__ == "__main__":
-    from text.chinese_bert import get_bert_feature
-
-    text = "啊！但是《原神》是由,米哈\游自主，  [研发]的一款全.新开放世界.冒险游戏"
-    text = text_normalize(text)
-    print(text)
-    phones, tones, word2ph = g2p(text)
-    bert = get_bert_feature(text, word2ph)
-
-    print(phones, tones, word2ph, bert.shape)
-
-
-# # 示例用法
-# text = "这是一个示例文本：,你好！这是一个测试...."
-# print(g2p_paddle(text))  # 输出: 这是一个示例文本你好这是一个测试

text/chinese_bert.py

@@ -1,119 +0,0 @@
-import sys
-
-import torch
-from transformers import AutoModelForMaskedLM, AutoTokenizer
-
-from config import config
-
-LOCAL_PATH = "./bert/chinese-roberta-wwm-ext-large"
-
-tokenizer = AutoTokenizer.from_pretrained(LOCAL_PATH)
-
-models = dict()
-
-
-def get_bert_feature(
-    text,
-    word2ph,
-    device=config.bert_gen_config.device,
-    style_text=None,
-    style_weight=0.7,
-):
-    if (
-        sys.platform == "darwin"
-        and torch.backends.mps.is_available()
-        and device == "cpu"
-    ):
-        device = "mps"
-    if not device:
-        device = "cuda"
-    if device not in models.keys():
-        models[device] = AutoModelForMaskedLM.from_pretrained(LOCAL_PATH).to(device)
-    with torch.no_grad():
-        inputs = tokenizer(text, return_tensors="pt")
-        for i in inputs:
-            inputs[i] = inputs[i].to(device)
-        res = models[device](**inputs, output_hidden_states=True)
-        res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
-        if style_text:
-            style_inputs = tokenizer(style_text, return_tensors="pt")
-            for i in style_inputs:
-                style_inputs[i] = style_inputs[i].to(device)
-            style_res = models[device](**style_inputs, output_hidden_states=True)
-            style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
-            style_res_mean = style_res.mean(0)
-    assert len(word2ph) == len(text) + 2
-    word2phone = word2ph
-    phone_level_feature = []
-    for i in range(len(word2phone)):
-        if style_text:
-            repeat_feature = (
-                res[i].repeat(word2phone[i], 1) * (1 - style_weight)
-                + style_res_mean.repeat(word2phone[i], 1) * style_weight
-            )
-        else:
-            repeat_feature = res[i].repeat(word2phone[i], 1)
-        phone_level_feature.append(repeat_feature)
-
-    phone_level_feature = torch.cat(phone_level_feature, dim=0)
-
-    return phone_level_feature.T
-
-
-if __name__ == "__main__":
-    word_level_feature = torch.rand(38, 1024)  # 12个词,每个词1024维特征
-    word2phone = [
-        1,
-        2,
-        1,
-        2,
-        2,
-        1,
-        2,
-        2,
-        1,
-        2,
-        2,
-        1,
-        2,
-        2,
-        2,
-        2,
-        2,
-        1,
-        1,
-        2,
-        2,
-        1,
-        2,
-        2,
-        2,
-        2,
-        1,
-        2,
-        2,
-        2,
-        2,
-        2,
-        1,
-        2,
-        2,
-        2,
-        2,
-        1,
-    ]
-
-    # 计算总帧数
-    total_frames = sum(word2phone)
-    print(word_level_feature.shape)
-    print(word2phone)
-    phone_level_feature = []
-    for i in range(len(word2phone)):
-        print(word_level_feature[i].shape)
-
-        # 对每个词重复word2phone[i]次
-        repeat_feature = word_level_feature[i].repeat(word2phone[i], 1)
-        phone_level_feature.append(repeat_feature)
-
-    phone_level_feature = torch.cat(phone_level_feature, dim=0)
-    print(phone_level_feature.shape)  # torch.Size([36, 1024])

text/cleaner.py

@@ -1,28 +0,0 @@
-from text import chinese, japanese, english, cleaned_text_to_sequence
-
-
-language_module_map = {"ZH": chinese, "JP": japanese, "EN": english}
-
-
-def clean_text(text, language):
-    language_module = language_module_map[language]
-    norm_text = language_module.text_normalize(text)
-    phones, tones, word2ph = language_module.g2p(norm_text)
-    return norm_text, phones, tones, word2ph
-
-
-def clean_text_bert(text, language):
-    language_module = language_module_map[language]
-    norm_text = language_module.text_normalize(text)
-    phones, tones, word2ph = language_module.g2p(norm_text)
-    bert = language_module.get_bert_feature(norm_text, word2ph)
-    return phones, tones, bert
-
-
-def text_to_sequence(text, language):
-    norm_text, phones, tones, word2ph = clean_text(text, language)
-    return cleaned_text_to_sequence(phones, tones, language)
-
-
-if __name__ == "__main__":
-    pass

text/cmudict_cache.pickle

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b9b21b20325471934ba92f2e4a5976989e7d920caa32e7a286eacb027d197949
-size 6212655

text/english.py

@@ -1,495 +0,0 @@
-import pickle
-import os
-import re
-from g2p_en import G2p
-from transformers import DebertaV2Tokenizer
-
-from text import symbols
-from text.symbols import punctuation
-
-current_file_path = os.path.dirname(__file__)
-CMU_DICT_PATH = os.path.join(current_file_path, "cmudict.rep")
-CACHE_PATH = os.path.join(current_file_path, "cmudict_cache.pickle")
-_g2p = G2p()
-LOCAL_PATH = "./bert/deberta-v3-large"
-tokenizer = DebertaV2Tokenizer.from_pretrained(LOCAL_PATH)
-
-arpa = {
-    "AH0",
-    "S",
-    "AH1",
-    "EY2",
-    "AE2",
-    "EH0",
-    "OW2",
-    "UH0",
-    "NG",
-    "B",
-    "G",
-    "AY0",
-    "M",
-    "AA0",
-    "F",
-    "AO0",
-    "ER2",
-    "UH1",
-    "IY1",
-    "AH2",
-    "DH",
-    "IY0",
-    "EY1",
-    "IH0",
-    "K",
-    "N",
-    "W",
-    "IY2",
-    "T",
-    "AA1",
-    "ER1",
-    "EH2",
-    "OY0",
-    "UH2",
-    "UW1",
-    "Z",
-    "AW2",
-    "AW1",
-    "V",
-    "UW2",
-    "AA2",
-    "ER",
-    "AW0",
-    "UW0",
-    "R",
-    "OW1",
-    "EH1",
-    "ZH",
-    "AE0",
-    "IH2",
-    "IH",
-    "Y",
-    "JH",
-    "P",
-    "AY1",
-    "EY0",
-    "OY2",
-    "TH",
-    "HH",
-    "D",
-    "ER0",
-    "CH",
-    "AO1",
-    "AE1",
-    "AO2",
-    "OY1",
-    "AY2",
-    "IH1",
-    "OW0",
-    "L",
-    "SH",
-}
-
-
-def post_replace_ph(ph):
-    rep_map = {
-        "：": ",",
-        "；": ",",
-        "，": ",",
-        "。": ".",
-        "！": "!",
-        "？": "?",
-        "\n": ".",
-        "·": ",",
-        "、": ",",
-        "…": "...",
-        "···": "...",
-        "・・・": "...",
-        "v": "V",
-    }
-    if ph in rep_map.keys():
-        ph = rep_map[ph]
-    if ph in symbols:
-        return ph
-    if ph not in symbols:
-        ph = "UNK"
-    return ph
-
-
-rep_map = {
-    "：": ",",
-    "；": ",",
-    "，": ",",
-    "。": ".",
-    "！": "!",
-    "？": "?",
-    "\n": ".",
-    "．": ".",
-    "…": "...",
-    "···": "...",
-    "・・・": "...",
-    "·": ",",
-    "・": ",",
-    "、": ",",
-    "$": ".",
-    "“": "'",
-    "”": "'",
-    '"': "'",
-    "‘": "'",
-    "’": "'",
-    "（": "'",
-    "）": "'",
-    "(": "'",
-    ")": "'",
-    "《": "'",
-    "》": "'",
-    "【": "'",
-    "】": "'",
-    "[": "'",
-    "]": "'",
-    "—": "-",
-    "−": "-",
-    "～": "-",
-    "~": "-",
-    "「": "'",
-    "」": "'",
-}
-
-
-def replace_punctuation(text):
-    pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
-
-    replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
-
-    # replaced_text = re.sub(
-    #     r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\u3005"
-    #     + "".join(punctuation)
-    #     + r"]+",
-    #     "",
-    #     replaced_text,
-    # )
-
-    return replaced_text
-
-
-def read_dict():
-    g2p_dict = {}
-    start_line = 49
-    with open(CMU_DICT_PATH) as f:
-        line = f.readline()
-        line_index = 1
-        while line:
-            if line_index >= start_line:
-                line = line.strip()
-                word_split = line.split("  ")
-                word = word_split[0]
-
-                syllable_split = word_split[1].split(" - ")
-                g2p_dict[word] = []
-                for syllable in syllable_split:
-                    phone_split = syllable.split(" ")
-                    g2p_dict[word].append(phone_split)
-
-            line_index = line_index + 1
-            line = f.readline()
-
-    return g2p_dict
-
-
-def cache_dict(g2p_dict, file_path):
-    with open(file_path, "wb") as pickle_file:
-        pickle.dump(g2p_dict, pickle_file)
-
-
-def get_dict():
-    if os.path.exists(CACHE_PATH):
-        with open(CACHE_PATH, "rb") as pickle_file:
-            g2p_dict = pickle.load(pickle_file)
-    else:
-        g2p_dict = read_dict()
-        cache_dict(g2p_dict, CACHE_PATH)
-
-    return g2p_dict
-
-
-eng_dict = get_dict()
-
-
-def refine_ph(phn):
-    tone = 0
-    if re.search(r"\d$", phn):
-        tone = int(phn[-1]) + 1
-        phn = phn[:-1]
-    else:
-        tone = 3
-    return phn.lower(), tone
-
-
-def refine_syllables(syllables):
-    tones = []
-    phonemes = []
-    for phn_list in syllables:
-        for i in range(len(phn_list)):
-            phn = phn_list[i]
-            phn, tone = refine_ph(phn)
-            phonemes.append(phn)
-            tones.append(tone)
-    return phonemes, tones
-
-
-import re
-import inflect
-
-_inflect = inflect.engine()
-_comma_number_re = re.compile(r"([0-9][0-9\,]+[0-9])")
-_decimal_number_re = re.compile(r"([0-9]+\.[0-9]+)")
-_pounds_re = re.compile(r"£([0-9\,]*[0-9]+)")
-_dollars_re = re.compile(r"\$([0-9\.\,]*[0-9]+)")
-_ordinal_re = re.compile(r"[0-9]+(st|nd|rd|th)")
-_number_re = re.compile(r"[0-9]+")
-
-# List of (regular expression, replacement) pairs for abbreviations:
-_abbreviations = [
-    (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
-    for x in [
-        ("mrs", "misess"),
-        ("mr", "mister"),
-        ("dr", "doctor"),
-        ("st", "saint"),
-        ("co", "company"),
-        ("jr", "junior"),
-        ("maj", "major"),
-        ("gen", "general"),
-        ("drs", "doctors"),
-        ("rev", "reverend"),
-        ("lt", "lieutenant"),
-        ("hon", "honorable"),
-        ("sgt", "sergeant"),
-        ("capt", "captain"),
-        ("esq", "esquire"),
-        ("ltd", "limited"),
-        ("col", "colonel"),
-        ("ft", "fort"),
-    ]
-]
-
-
-# List of (ipa, lazy ipa) pairs:
-_lazy_ipa = [
-    (re.compile("%s" % x[0]), x[1])
-    for x in [
-        ("r", "ɹ"),
-        ("æ", "e"),
-        ("ɑ", "a"),
-        ("ɔ", "o"),
-        ("ð", "z"),
-        ("θ", "s"),
-        ("ɛ", "e"),
-        ("ɪ", "i"),
-        ("ʊ", "u"),
-        ("ʒ", "ʥ"),
-        ("ʤ", "ʥ"),
-        ("ˈ", "↓"),
-    ]
-]
-
-# List of (ipa, lazy ipa2) pairs:
-_lazy_ipa2 = [
-    (re.compile("%s" % x[0]), x[1])
-    for x in [
-        ("r", "ɹ"),
-        ("ð", "z"),
-        ("θ", "s"),
-        ("ʒ", "ʑ"),
-        ("ʤ", "dʑ"),
-        ("ˈ", "↓"),
-    ]
-]
-
-# List of (ipa, ipa2) pairs
-_ipa_to_ipa2 = [
-    (re.compile("%s" % x[0]), x[1]) for x in [("r", "ɹ"), ("ʤ", "dʒ"), ("ʧ", "tʃ")]
-]
-
-
-def _expand_dollars(m):
-    match = m.group(1)
-    parts = match.split(".")
-    if len(parts) > 2:
-        return match + " dollars"  # Unexpected format
-    dollars = int(parts[0]) if parts[0] else 0
-    cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0
-    if dollars and cents:
-        dollar_unit = "dollar" if dollars == 1 else "dollars"
-        cent_unit = "cent" if cents == 1 else "cents"
-        return "%s %s, %s %s" % (dollars, dollar_unit, cents, cent_unit)
-    elif dollars:
-        dollar_unit = "dollar" if dollars == 1 else "dollars"
-        return "%s %s" % (dollars, dollar_unit)
-    elif cents:
-        cent_unit = "cent" if cents == 1 else "cents"
-        return "%s %s" % (cents, cent_unit)
-    else:
-        return "zero dollars"
-
-
-def _remove_commas(m):
-    return m.group(1).replace(",", "")
-
-
-def _expand_ordinal(m):
-    return _inflect.number_to_words(m.group(0))
-
-
-def _expand_number(m):
-    num = int(m.group(0))
-    if num > 1000 and num < 3000:
-        if num == 2000:
-            return "two thousand"
-        elif num > 2000 and num < 2010:
-            return "two thousand " + _inflect.number_to_words(num % 100)
-        elif num % 100 == 0:
-            return _inflect.number_to_words(num // 100) + " hundred"
-        else:
-            return _inflect.number_to_words(
-                num, andword="", zero="oh", group=2
-            ).replace(", ", " ")
-    else:
-        return _inflect.number_to_words(num, andword="")
-
-
-def _expand_decimal_point(m):
-    return m.group(1).replace(".", " point ")
-
-
-def normalize_numbers(text):
-    text = re.sub(_comma_number_re, _remove_commas, text)
-    text = re.sub(_pounds_re, r"\1 pounds", text)
-    text = re.sub(_dollars_re, _expand_dollars, text)
-    text = re.sub(_decimal_number_re, _expand_decimal_point, text)
-    text = re.sub(_ordinal_re, _expand_ordinal, text)
-    text = re.sub(_number_re, _expand_number, text)
-    return text
-
-
-def text_normalize(text):
-    text = normalize_numbers(text)
-    text = replace_punctuation(text)
-    text = re.sub(r"([,;.\?\!])([\w])", r"\1 \2", text)
-    return text
-
-
-def distribute_phone(n_phone, n_word):
-    phones_per_word = [0] * n_word
-    for task in range(n_phone):
-        min_tasks = min(phones_per_word)
-        min_index = phones_per_word.index(min_tasks)
-        phones_per_word[min_index] += 1
-    return phones_per_word
-
-
-def sep_text(text):
-    words = re.split(r"([,;.\?\!\s+])", text)
-    words = [word for word in words if word.strip() != ""]
-    return words
-
-
-def text_to_words(text):
-    tokens = tokenizer.tokenize(text)
-    words = []
-    for idx, t in enumerate(tokens):
-        if t.startswith("▁"):
-            words.append([t[1:]])
-        else:
-            if t in punctuation:
-                if idx == len(tokens) - 1:
-                    words.append([f"{t}"])
-                else:
-                    if (
-                        not tokens[idx + 1].startswith("▁")
-                        and tokens[idx + 1] not in punctuation
-                    ):
-                        if idx == 0:
-                            words.append([])
-                        words[-1].append(f"{t}")
-                    else:
-                        words.append([f"{t}"])
-            else:
-                if idx == 0:
-                    words.append([])
-                words[-1].append(f"{t}")
-    return words
-
-
-def g2p(text):
-    phones = []
-    tones = []
-    phone_len = []
-    # words = sep_text(text)
-    # tokens = [tokenizer.tokenize(i) for i in words]
-    words = text_to_words(text)
-
-    for word in words:
-        temp_phones, temp_tones = [], []
-        if len(word) > 1:
-            if "'" in word:
-                word = ["".join(word)]
-        for w in word:
-            if w in punctuation:
-                temp_phones.append(w)
-                temp_tones.append(0)
-                continue
-            if w.upper() in eng_dict:
-                phns, tns = refine_syllables(eng_dict[w.upper()])
-                temp_phones += [post_replace_ph(i) for i in phns]
-                temp_tones += tns
-                # w2ph.append(len(phns))
-            else:
-                phone_list = list(filter(lambda p: p != " ", _g2p(w)))
-                phns = []
-                tns = []
-                for ph in phone_list:
-                    if ph in arpa:
-                        ph, tn = refine_ph(ph)
-                        phns.append(ph)
-                        tns.append(tn)
-                    else:
-                        phns.append(ph)
-                        tns.append(0)
-                temp_phones += [post_replace_ph(i) for i in phns]
-                temp_tones += tns
-        phones += temp_phones
-        tones += temp_tones
-        phone_len.append(len(temp_phones))
-        # phones = [post_replace_ph(i) for i in phones]
-
-    word2ph = []
-    for token, pl in zip(words, phone_len):
-        word_len = len(token)
-
-        aaa = distribute_phone(pl, word_len)
-        word2ph += aaa
-
-    phones = ["_"] + phones + ["_"]
-    tones = [0] + tones + [0]
-    word2ph = [1] + word2ph + [1]
-    assert len(phones) == len(tones), text
-    assert len(phones) == sum(word2ph), text
-
-    return phones, tones, word2ph
-
-
-def get_bert_feature(text, word2ph):
-    from text import english_bert_mock
-
-    return english_bert_mock.get_bert_feature(text, word2ph)
-
-
-if __name__ == "__main__":
-    # print(get_dict())
-    # print(eng_word_to_phoneme("hello"))
-    print(g2p("In this paper, we propose 1 DSPGAN, a GAN-based universal vocoder."))
-    # all_phones = set()
-    # for k, syllables in eng_dict.items():
-    #     for group in syllables:
-    #         for ph in group:
-    #             all_phones.add(ph)
-    # print(all_phones)

text/english_bert_mock.py

@@ -1,61 +0,0 @@
-import sys
-
-import torch
-from transformers import DebertaV2Model, DebertaV2Tokenizer
-
-from config import config
-
-
-LOCAL_PATH = "./bert/deberta-v3-large"
-
-tokenizer = DebertaV2Tokenizer.from_pretrained(LOCAL_PATH)
-
-models = dict()
-
-
-def get_bert_feature(
-    text,
-    word2ph,
-    device=config.bert_gen_config.device,
-    style_text=None,
-    style_weight=0.7,
-):
-    if (
-        sys.platform == "darwin"
-        and torch.backends.mps.is_available()
-        and device == "cpu"
-    ):
-        device = "mps"
-    if not device:
-        device = "cuda"
-    if device not in models.keys():
-        models[device] = DebertaV2Model.from_pretrained(LOCAL_PATH).to(device)
-    with torch.no_grad():
-        inputs = tokenizer(text, return_tensors="pt")
-        for i in inputs:
-            inputs[i] = inputs[i].to(device)
-        res = models[device](**inputs, output_hidden_states=True)
-        res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
-        if style_text:
-            style_inputs = tokenizer(style_text, return_tensors="pt")
-            for i in style_inputs:
-                style_inputs[i] = style_inputs[i].to(device)
-            style_res = models[device](**style_inputs, output_hidden_states=True)
-            style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
-            style_res_mean = style_res.mean(0)
-    assert len(word2ph) == res.shape[0], (text, res.shape[0], len(word2ph))
-    word2phone = word2ph
-    phone_level_feature = []
-    for i in range(len(word2phone)):
-        if style_text:
-            repeat_feature = (
-                res[i].repeat(word2phone[i], 1) * (1 - style_weight)
-                + style_res_mean.repeat(word2phone[i], 1) * style_weight
-            )
-        else:
-            repeat_feature = res[i].repeat(word2phone[i], 1)
-        phone_level_feature.append(repeat_feature)
-
-    phone_level_feature = torch.cat(phone_level_feature, dim=0)
-
-    return phone_level_feature.T

text/japanese.py

@@ -1,432 +0,0 @@
-# Convert Japanese text to phonemes which is
-# compatible with Julius https://github.com/julius-speech/segmentation-kit
-import re
-import unicodedata
-
-from transformers import AutoTokenizer
-
-from text import punctuation, symbols
-
-from num2words import num2words
-
-import pyopenjtalk
-import jaconv
-
-
-def kata2phoneme(text: str) -> str:
-    """Convert katakana text to phonemes."""
-    text = text.strip()
-    if text == "ー":
-        return ["ー"]
-    elif text.startswith("ー"):
-        return ["ー"] + kata2phoneme(text[1:])
-    res = []
-    prev = None
-    while text:
-        if re.match(_MARKS, text):
-            res.append(text)
-            text = text[1:]
-            continue
-        if text.startswith("ー"):
-            if prev:
-                res.append(prev[-1])
-            text = text[1:]
-            continue
-        res += pyopenjtalk.g2p(text).lower().replace("cl", "q").split(" ")
-        break
-    # res = _COLON_RX.sub(":", res)
-    return res
-
-
-def hira2kata(text: str) -> str:
-    return jaconv.hira2kata(text)
-
-
-_SYMBOL_TOKENS = set(list("・、。？！"))
-_NO_YOMI_TOKENS = set(list("「」『』―（）［］[]"))
-_MARKS = re.compile(
-    r"[^A-Za-z\d\u3005\u3040-\u30ff\u4e00-\u9fff\uff11-\uff19\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d]"
-)
-
-
-def text2kata(text: str) -> str:
-    parsed = pyopenjtalk.run_frontend(text)
-
-    res = []
-    for parts in parsed:
-        word, yomi = replace_punctuation(parts["string"]), parts["pron"].replace(
-            "’", ""
-        )
-        if yomi:
-            if re.match(_MARKS, yomi):
-                if len(word) > 1:
-                    word = [replace_punctuation(i) for i in list(word)]
-                    yomi = word
-                    res += yomi
-                    sep += word
-                    continue
-                elif word not in rep_map.keys() and word not in rep_map.values():
-                    word = ","
-                yomi = word
-            res.append(yomi)
-        else:
-            if word in _SYMBOL_TOKENS:
-                res.append(word)
-            elif word in ("っ", "ッ"):
-                res.append("ッ")
-            elif word in _NO_YOMI_TOKENS:
-                pass
-            else:
-                res.append(word)
-    return hira2kata("".join(res))
-
-
-def text2sep_kata(text: str) -> (list, list):
-    parsed = pyopenjtalk.run_frontend(text)
-
-    res = []
-    sep = []
-    for parts in parsed:
-        word, yomi = replace_punctuation(parts["string"]), parts["pron"].replace(
-            "’", ""
-        )
-        if yomi:
-            if re.match(_MARKS, yomi):
-                if len(word) > 1:
-                    word = [replace_punctuation(i) for i in list(word)]
-                    yomi = word
-                    res += yomi
-                    sep += word
-                    continue
-                elif word not in rep_map.keys() and word not in rep_map.values():
-                    word = ","
-                yomi = word
-            res.append(yomi)
-        else:
-            if word in _SYMBOL_TOKENS:
-                res.append(word)
-            elif word in ("っ", "ッ"):
-                res.append("ッ")
-            elif word in _NO_YOMI_TOKENS:
-                pass
-            else:
-                res.append(word)
-        sep.append(word)
-    return sep, [hira2kata(i) for i in res], get_accent(parsed)
-
-
-def get_accent(parsed):
-    labels = pyopenjtalk.make_label(parsed)
-
-    phonemes = []
-    accents = []
-    for n, label in enumerate(labels):
-        phoneme = re.search(r"\-([^\+]*)\+", label).group(1)
-        if phoneme not in ["sil", "pau"]:
-            phonemes.append(phoneme.replace("cl", "q").lower())
-        else:
-            continue
-        a1 = int(re.search(r"/A:(\-?[0-9]+)\+", label).group(1))
-        a2 = int(re.search(r"\+(\d+)\+", label).group(1))
-        if re.search(r"\-([^\+]*)\+", labels[n + 1]).group(1) in ["sil", "pau"]:
-            a2_next = -1
-        else:
-            a2_next = int(re.search(r"\+(\d+)\+", labels[n + 1]).group(1))
-        # Falling
-        if a1 == 0 and a2_next == a2 + 1:
-            accents.append(-1)
-        # Rising
-        elif a2 == 1 and a2_next == 2:
-            accents.append(1)
-        else:
-            accents.append(0)
-    return list(zip(phonemes, accents))
-
-
-_ALPHASYMBOL_YOMI = {
-    "#": "シャープ",
-    "%": "パーセント",
-    "&": "アンド",
-    "+": "プラス",
-    "-": "マイナス",
-    ":": "コロン",
-    ";": "セミコロン",
-    "<": "小なり",
-    "=": "イコール",
-    ">": "大なり",
-    "@": "アット",
-    "a": "エー",
-    "b": "ビー",
-    "c": "シー",
-    "d": "ディー",
-    "e": "イー",
-    "f": "エフ",
-    "g": "ジー",
-    "h": "エイチ",
-    "i": "アイ",
-    "j": "ジェー",
-    "k": "ケー",
-    "l": "エル",
-    "m": "エム",
-    "n": "エヌ",
-    "o": "オー",
-    "p": "ピー",
-    "q": "キュー",
-    "r": "アール",
-    "s": "エス",
-    "t": "ティー",
-    "u": "ユー",
-    "v": "ブイ",
-    "w": "ダブリュー",
-    "x": "エックス",
-    "y": "ワイ",
-    "z": "ゼット",
-    "α": "アルファ",
-    "β": "ベータ",
-    "γ": "ガンマ",
-    "δ": "デルタ",
-    "ε": "イプシロン",
-    "ζ": "ゼータ",
-    "η": "イータ",
-    "θ": "シータ",
-    "ι": "イオタ",
-    "κ": "カッパ",
-    "λ": "ラムダ",
-    "μ": "ミュー",
-    "ν": "ニュー",
-    "ξ": "クサイ",
-    "ο": "オミクロン",
-    "π": "パイ",
-    "ρ": "ロー",
-    "σ": "シグマ",
-    "τ": "タウ",
-    "υ": "ウプシロン",
-    "φ": "ファイ",
-    "χ": "カイ",
-    "ψ": "プサイ",
-    "ω": "オメガ",
-}
-
-
-_NUMBER_WITH_SEPARATOR_RX = re.compile("[0-9]{1,3}(,[0-9]{3})+")
-_CURRENCY_MAP = {"$": "ドル", "¥": "円", "£": "ポンド", "€": "ユーロ"}
-_CURRENCY_RX = re.compile(r"([$¥£€])([0-9.]*[0-9])")
-_NUMBER_RX = re.compile(r"[0-9]+(\.[0-9]+)?")
-
-
-def japanese_convert_numbers_to_words(text: str) -> str:
-    res = _NUMBER_WITH_SEPARATOR_RX.sub(lambda m: m[0].replace(",", ""), text)
-    res = _CURRENCY_RX.sub(lambda m: m[2] + _CURRENCY_MAP.get(m[1], m[1]), res)
-    res = _NUMBER_RX.sub(lambda m: num2words(m[0], lang="ja"), res)
-    return res
-
-
-def japanese_convert_alpha_symbols_to_words(text: str) -> str:
-    return "".join([_ALPHASYMBOL_YOMI.get(ch, ch) for ch in text.lower()])
-
-
-def japanese_text_to_phonemes(text: str) -> str:
-    """Convert Japanese text to phonemes."""
-    res = unicodedata.normalize("NFKC", text)
-    res = japanese_convert_numbers_to_words(res)
-    # res = japanese_convert_alpha_symbols_to_words(res)
-    res = text2kata(res)
-    res = kata2phoneme(res)
-    return res
-
-
-def is_japanese_character(char):
-    # 定义日语文字系统的 Unicode 范围
-    japanese_ranges = [
-        (0x3040, 0x309F),  # 平假名
-        (0x30A0, 0x30FF),  # 片假名
-        (0x4E00, 0x9FFF),  # 汉字 (CJK Unified Ideographs)
-        (0x3400, 0x4DBF),  # 汉字扩展 A
-        (0x20000, 0x2A6DF),  # 汉字扩展 B
-        # 可以根据需要添加其他汉字扩展范围
-    ]
-
-    # 将字符的 Unicode 编码转换为整数
-    char_code = ord(char)
-
-    # 检查字符是否在任何一个日语范围内
-    for start, end in japanese_ranges:
-        if start <= char_code <= end:
-            return True
-
-    return False
-
-
-rep_map = {
-    "：": ",",
-    "；": ",",
-    "，": ",",
-    "。": ".",
-    "！": "!",
-    "？": "?",
-    "\n": ".",
-    "．": ".",
-    "…": "...",
-    "···": "...",
-    "・・・": "...",
-    "·": ",",
-    "・": ",",
-    "、": ",",
-    "$": ".",
-    "“": "'",
-    "”": "'",
-    '"': "'",
-    "‘": "'",
-    "’": "'",
-    "（": "'",
-    "）": "'",
-    "(": "'",
-    ")": "'",
-    "《": "'",
-    "》": "'",
-    "【": "'",
-    "】": "'",
-    "[": "'",
-    "]": "'",
-    "—": "-",
-    "−": "-",
-    "～": "-",
-    "~": "-",
-    "「": "'",
-    "」": "'",
-}
-
-
-def replace_punctuation(text):
-    pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
-
-    replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
-
-    replaced_text = re.sub(
-        r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\u3005"
-        + "".join(punctuation)
-        + r"]+",
-        "",
-        replaced_text,
-    )
-
-    return replaced_text
-
-
-def text_normalize(text):
-    res = unicodedata.normalize("NFKC", text)
-    res = japanese_convert_numbers_to_words(res)
-    # res = "".join([i for i in res if is_japanese_character(i)])
-    res = replace_punctuation(res)
-    res = res.replace("゙", "")
-    return res
-
-
-def distribute_phone(n_phone, n_word):
-    phones_per_word = [0] * n_word
-    for task in range(n_phone):
-        min_tasks = min(phones_per_word)
-        min_index = phones_per_word.index(min_tasks)
-        phones_per_word[min_index] += 1
-    return phones_per_word
-
-
-def handle_long(sep_phonemes):
-    for i in range(len(sep_phonemes)):
-        if sep_phonemes[i][0] == "ー":
-            sep_phonemes[i][0] = sep_phonemes[i - 1][-1]
-        if "ー" in sep_phonemes[i]:
-            for j in range(len(sep_phonemes[i])):
-                if sep_phonemes[i][j] == "ー":
-                    sep_phonemes[i][j] = sep_phonemes[i][j - 1][-1]
-    return sep_phonemes
-
-
-tokenizer = AutoTokenizer.from_pretrained("./bert/deberta-v2-large-japanese-char-wwm")
-
-
-def align_tones(phones, tones):
-    res = []
-    for pho in phones:
-        temp = [0] * len(pho)
-        for idx, p in enumerate(pho):
-            if len(tones) == 0:
-                break
-            if p == tones[0][0]:
-                temp[idx] = tones[0][1]
-                if idx > 0:
-                    temp[idx] += temp[idx - 1]
-                tones.pop(0)
-        temp = [0] + temp
-        temp = temp[:-1]
-        if -1 in temp:
-            temp = [i + 1 for i in temp]
-        res.append(temp)
-    res = [i for j in res for i in j]
-    assert not any([i < 0 for i in res]) and not any([i > 1 for i in res])
-    return res
-
-
-def rearrange_tones(tones, phones):
-    res = [0] * len(tones)
-    for i in range(len(tones)):
-        if i == 0:
-            if tones[i] not in punctuation:
-                res[i] = 1
-        elif tones[i] == prev:
-            if phones[i] in punctuation:
-                res[i] = 0
-            else:
-                res[i] = 1
-        elif tones[i] > prev:
-            res[i] = 2
-        elif tones[i] < prev:
-            res[i - 1] = 3
-            res[i] = 1
-        prev = tones[i]
-    return res
-
-
-def g2p(norm_text):
-    sep_text, sep_kata, acc = text2sep_kata(norm_text)
-    sep_tokenized = []
-    for i in sep_text:
-        if i not in punctuation:
-            sep_tokenized.append(tokenizer.tokenize(i))
-        else:
-            sep_tokenized.append([i])
-
-    sep_phonemes = handle_long([kata2phoneme(i) for i in sep_kata])
-    # 异常处理，MeCab不认识的词的话会一路传到这里来，然后炸掉。目前来看只有那些超级稀有的生僻词会出现这种情况
-    for i in sep_phonemes:
-        for j in i:
-            assert j in symbols, (sep_text, sep_kata, sep_phonemes)
-    tones = align_tones(sep_phonemes, acc)
-
-    word2ph = []
-    for token, phoneme in zip(sep_tokenized, sep_phonemes):
-        phone_len = len(phoneme)
-        word_len = len(token)
-
-        aaa = distribute_phone(phone_len, word_len)
-        word2ph += aaa
-    phones = ["_"] + [j for i in sep_phonemes for j in i] + ["_"]
-    # tones = [0] + rearrange_tones(tones, phones[1:-1]) + [0]
-    tones = [0] + tones + [0]
-    word2ph = [1] + word2ph + [1]
-    assert len(phones) == len(tones)
-    return phones, tones, word2ph
-
-
-if __name__ == "__main__":
-    tokenizer = AutoTokenizer.from_pretrained("./bert/deberta-v2-large-japanese")
-    text = "hello,こんにちは、世界ー！……"
-    from text.japanese_bert import get_bert_feature
-
-    text = text_normalize(text)
-    print(text)
-
-    phones, tones, word2ph = g2p(text)
-    bert = get_bert_feature(text, word2ph)
-
-    print(phones, tones, word2ph, bert.shape)

text/japanese_bert.py

@@ -1,65 +0,0 @@
-import sys
-
-import torch
-from transformers import AutoModelForMaskedLM, AutoTokenizer
-
-from config import config
-from text.japanese import text2sep_kata
-
-LOCAL_PATH = "./bert/deberta-v2-large-japanese-char-wwm"
-
-tokenizer = AutoTokenizer.from_pretrained(LOCAL_PATH)
-
-models = dict()
-
-
-def get_bert_feature(
-    text,
-    word2ph,
-    device=config.bert_gen_config.device,
-    style_text=None,
-    style_weight=0.7,
-):
-    text = "".join(text2sep_kata(text)[0])
-    if style_text:
-        style_text = "".join(text2sep_kata(style_text)[0])
-    if (
-        sys.platform == "darwin"
-        and torch.backends.mps.is_available()
-        and device == "cpu"
-    ):
-        device = "mps"
-    if not device:
-        device = "cuda"
-    if device not in models.keys():
-        models[device] = AutoModelForMaskedLM.from_pretrained(LOCAL_PATH).to(device)
-    with torch.no_grad():
-        inputs = tokenizer(text, return_tensors="pt")
-        for i in inputs:
-            inputs[i] = inputs[i].to(device)
-        res = models[device](**inputs, output_hidden_states=True)
-        res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
-        if style_text:
-            style_inputs = tokenizer(style_text, return_tensors="pt")
-            for i in style_inputs:
-                style_inputs[i] = style_inputs[i].to(device)
-            style_res = models[device](**style_inputs, output_hidden_states=True)
-            style_res = torch.cat(style_res["hidden_states"][-3:-2], -1)[0].cpu()
-            style_res_mean = style_res.mean(0)
-
-    assert len(word2ph) == len(text) + 2
-    word2phone = word2ph
-    phone_level_feature = []
-    for i in range(len(word2phone)):
-        if style_text:
-            repeat_feature = (
-                res[i].repeat(word2phone[i], 1) * (1 - style_weight)
-                + style_res_mean.repeat(word2phone[i], 1) * style_weight
-            )
-        else:
-            repeat_feature = res[i].repeat(word2phone[i], 1)
-        phone_level_feature.append(repeat_feature)
-
-    phone_level_feature = torch.cat(phone_level_feature, dim=0)
-
-    return phone_level_feature.T

text/opencpop-strict.txt

@@ -1,429 +0,0 @@
-a	AA a
-ai	AA ai
-an	AA an
-ang	AA ang
-ao	AA ao
-ba	b a
-bai	b ai
-ban	b an
-bang	b ang
-bao	b ao
-bei	b ei
-ben	b en
-beng	b eng
-bi	b i
-bian	b ian
-biao	b iao
-bie	b ie
-bin	b in
-bing	b ing
-bo	b o
-bu	b u
-ca	c a
-cai	c ai
-can	c an
-cang	c ang
-cao	c ao
-ce	c e
-cei	c ei
-cen	c en
-ceng	c eng
-cha	ch a
-chai	ch ai
-chan	ch an
-chang	ch ang
-chao	ch ao
-che	ch e
-chen	ch en
-cheng	ch eng
-chi	ch ir
-chong	ch ong
-chou	ch ou
-chu	ch u
-chua	ch ua
-chuai	ch uai
-chuan	ch uan
-chuang	ch uang
-chui	ch ui
-chun	ch un
-chuo	ch uo
-ci	c i0
-cong	c ong
-cou	c ou
-cu	c u
-cuan	c uan
-cui	c ui
-cun	c un
-cuo	c uo
-da	d a
-dai	d ai
-dan	d an
-dang	d ang
-dao	d ao
-de	d e
-dei	d ei
-den	d en
-deng	d eng
-di	d i
-dia	d ia
-dian	d ian
-diao	d iao
-die	d ie
-ding	d ing
-diu	d iu
-dong	d ong
-dou	d ou
-du	d u
-duan	d uan
-dui	d ui
-dun	d un
-duo	d uo
-e	EE e
-ei	EE ei
-en	EE en
-eng	EE eng
-er	EE er
-fa	f a
-fan	f an
-fang	f ang
-fei	f ei
-fen	f en
-feng	f eng
-fo	f o
-fou	f ou
-fu	f u
-ga	g a
-gai	g ai
-gan	g an
-gang	g ang
-gao	g ao
-ge	g e
-gei	g ei
-gen	g en
-geng	g eng
-gong	g ong
-gou	g ou
-gu	g u
-gua	g ua
-guai	g uai
-guan	g uan
-guang	g uang
-gui	g ui
-gun	g un
-guo	g uo
-ha	h a
-hai	h ai
-han	h an
-hang	h ang
-hao	h ao
-he	h e
-hei	h ei
-hen	h en
-heng	h eng
-hong	h ong
-hou	h ou
-hu	h u
-hua	h ua
-huai	h uai
-huan	h uan
-huang	h uang
-hui	h ui
-hun	h un
-huo	h uo
-ji	j i
-jia	j ia
-jian	j ian
-jiang	j iang
-jiao	j iao
-jie	j ie
-jin	j in
-jing	j ing
-jiong	j iong
-jiu	j iu
-ju	j v
-jv	j v
-juan	j van
-jvan	j van
-jue	j ve
-jve	j ve
-jun	j vn
-jvn	j vn
-ka	k a
-kai	k ai
-kan	k an
-kang	k ang
-kao	k ao
-ke	k e
-kei	k ei
-ken	k en
-keng	k eng
-kong	k ong
-kou	k ou
-ku	k u
-kua	k ua
-kuai	k uai
-kuan	k uan
-kuang	k uang
-kui	k ui
-kun	k un
-kuo	k uo
-la	l a
-lai	l ai
-lan	l an
-lang	l ang
-lao	l ao
-le	l e
-lei	l ei
-leng	l eng
-li	l i
-lia	l ia
-lian	l ian
-liang	l iang
-liao	l iao
-lie	l ie
-lin	l in
-ling	l ing
-liu	l iu
-lo	l o
-long	l ong
-lou	l ou
-lu	l u
-luan	l uan
-lun	l un
-luo	l uo
-lv	l v
-lve	l ve
-ma	m a
-mai	m ai
-man	m an
-mang	m ang
-mao	m ao
-me	m e
-mei	m ei
-men	m en
-meng	m eng
-mi	m i
-mian	m ian
-miao	m iao
-mie	m ie
-min	m in
-ming	m ing
-miu	m iu
-mo	m o
-mou	m ou
-mu	m u
-na	n a
-nai	n ai
-nan	n an
-nang	n ang
-nao	n ao
-ne	n e
-nei	n ei
-nen	n en
-neng	n eng
-ni	n i
-nian	n ian
-niang	n iang
-niao	n iao
-nie	n ie
-nin	n in
-ning	n ing
-niu	n iu
-nong	n ong
-nou	n ou
-nu	n u
-nuan	n uan
-nun	n un
-nuo	n uo
-nv	n v
-nve	n ve
-o	OO o
-ou	OO ou
-pa	p a
-pai	p ai
-pan	p an
-pang	p ang
-pao	p ao
-pei	p ei
-pen	p en
-peng	p eng
-pi	p i
-pian	p ian
-piao	p iao
-pie	p ie
-pin	p in
-ping	p ing
-po	p o
-pou	p ou
-pu	p u
-qi	q i
-qia	q ia
-qian	q ian
-qiang	q iang
-qiao	q iao
-qie	q ie
-qin	q in
-qing	q ing
-qiong	q iong
-qiu	q iu
-qu	q v
-qv	q v
-quan	q van
-qvan	q van
-que	q ve
-qve	q ve
-qun	q vn
-qvn	q vn
-ran	r an
-rang	r ang
-rao	r ao
-re	r e
-ren	r en
-reng	r eng
-ri	r ir
-rong	r ong
-rou	r ou
-ru	r u
-rua	r ua
-ruan	r uan
-rui	r ui
-run	r un
-ruo	r uo
-sa	s a
-sai	s ai
-san	s an
-sang	s ang
-sao	s ao
-se	s e
-sen	s en
-seng	s eng
-sha	sh a
-shai	sh ai
-shan	sh an
-shang	sh ang
-shao	sh ao
-she	sh e
-shei	sh ei
-shen	sh en
-sheng	sh eng
-shi	sh ir
-shou	sh ou
-shu	sh u
-shua	sh ua
-shuai	sh uai
-shuan	sh uan
-shuang	sh uang
-shui	sh ui
-shun	sh un
-shuo	sh uo
-si	s i0
-song	s ong
-sou	s ou
-su	s u
-suan	s uan
-sui	s ui
-sun	s un
-suo	s uo
-ta	t a
-tai	t ai
-tan	t an
-tang	t ang
-tao	t ao
-te	t e
-tei	t ei
-teng	t eng
-ti	t i
-tian	t ian
-tiao	t iao
-tie	t ie
-ting	t ing
-tong	t ong
-tou	t ou
-tu	t u
-tuan	t uan
-tui	t ui
-tun	t un
-tuo	t uo
-wa	w a
-wai	w ai
-wan	w an
-wang	w ang
-wei	w ei
-wen	w en
-weng	w eng
-wo	w o
-wu	w u
-xi	x i
-xia	x ia
-xian	x ian
-xiang	x iang
-xiao	x iao
-xie	x ie
-xin	x in
-xing	x ing
-xiong	x iong
-xiu	x iu
-xu	x v
-xv	x v
-xuan	x van
-xvan	x van
-xue	x ve
-xve	x ve
-xun	x vn
-xvn	x vn
-ya	y a
-yan	y En
-yang	y ang
-yao	y ao
-ye	y E
-yi	y i
-yin	y in
-ying	y ing
-yo	y o
-yong	y ong
-you	y ou
-yu	y v
-yv	y v
-yuan	y van
-yvan	y van
-yue	y ve
-yve	y ve
-yun	y vn
-yvn	y vn
-za	z a
-zai	z ai
-zan	z an
-zang	z ang
-zao	z ao
-ze	z e
-zei	z ei
-zen	z en
-zeng	z eng
-zha	zh a
-zhai	zh ai
-zhan	zh an
-zhang	zh ang
-zhao	zh ao
-zhe	zh e
-zhei	zh ei
-zhen	zh en
-zheng	zh eng
-zhi	zh ir
-zhong	zh ong
-zhou	zh ou
-zhu	zh u
-zhua	zh ua
-zhuai	zh uai
-zhuan	zh uan
-zhuang	zh uang
-zhui	zh ui
-zhun	zh un
-zhuo	zh uo
-zi	z i0
-zong	z ong
-zou	z ou
-zu	z u
-zuan	z uan
-zui	z ui
-zun	z un
-zuo	z uo

text/symbols.py

@@ -1,187 +0,0 @@
-punctuation = ["!", "?", "…", ",", ".", "'", "-"]
-pu_symbols = punctuation + ["SP", "UNK"]
-pad = "_"
-
-# chinese
-zh_symbols = [
-    "E",
-    "En",
-    "a",
-    "ai",
-    "an",
-    "ang",
-    "ao",
-    "b",
-    "c",
-    "ch",
-    "d",
-    "e",
-    "ei",
-    "en",
-    "eng",
-    "er",
-    "f",
-    "g",
-    "h",
-    "i",
-    "i0",
-    "ia",
-    "ian",
-    "iang",
-    "iao",
-    "ie",
-    "in",
-    "ing",
-    "iong",
-    "ir",
-    "iu",
-    "j",
-    "k",
-    "l",
-    "m",
-    "n",
-    "o",
-    "ong",
-    "ou",
-    "p",
-    "q",
-    "r",
-    "s",
-    "sh",
-    "t",
-    "u",
-    "ua",
-    "uai",
-    "uan",
-    "uang",
-    "ui",
-    "un",
-    "uo",
-    "v",
-    "van",
-    "ve",
-    "vn",
-    "w",
-    "x",
-    "y",
-    "z",
-    "zh",
-    "AA",
-    "EE",
-    "OO",
-]
-num_zh_tones = 6
-
-# japanese
-ja_symbols = [
-    "N",
-    "a",
-    "a:",
-    "b",
-    "by",
-    "ch",
-    "d",
-    "dy",
-    "e",
-    "e:",
-    "f",
-    "g",
-    "gy",
-    "h",
-    "hy",
-    "i",
-    "i:",
-    "j",
-    "k",
-    "ky",
-    "m",
-    "my",
-    "n",
-    "ny",
-    "o",
-    "o:",
-    "p",
-    "py",
-    "q",
-    "r",
-    "ry",
-    "s",
-    "sh",
-    "t",
-    "ts",
-    "ty",
-    "u",
-    "u:",
-    "w",
-    "y",
-    "z",
-    "zy",
-]
-num_ja_tones = 2
-
-# English
-en_symbols = [
-    "aa",
-    "ae",
-    "ah",
-    "ao",
-    "aw",
-    "ay",
-    "b",
-    "ch",
-    "d",
-    "dh",
-    "eh",
-    "er",
-    "ey",
-    "f",
-    "g",
-    "hh",
-    "ih",
-    "iy",
-    "jh",
-    "k",
-    "l",
-    "m",
-    "n",
-    "ng",
-    "ow",
-    "oy",
-    "p",
-    "r",
-    "s",
-    "sh",
-    "t",
-    "th",
-    "uh",
-    "uw",
-    "V",
-    "w",
-    "y",
-    "z",
-    "zh",
-]
-num_en_tones = 4
-
-# combine all symbols
-normal_symbols = sorted(set(zh_symbols + ja_symbols + en_symbols))
-symbols = [pad] + normal_symbols + pu_symbols
-sil_phonemes_ids = [symbols.index(i) for i in pu_symbols]
-
-# combine all tones
-num_tones = num_zh_tones + num_ja_tones + num_en_tones
-
-# language maps
-language_id_map = {"ZH": 0, "JP": 1, "EN": 2}
-num_languages = len(language_id_map.keys())
-
-language_tone_start_map = {
-    "ZH": 0,
-    "JP": num_zh_tones,
-    "EN": num_zh_tones + num_ja_tones,
-}
-
-if __name__ == "__main__":
-    a = set(zh_symbols)
-    b = set(en_symbols)
-    print(sorted(a & b))

text/tone_sandhi.py

@@ -1,773 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from typing import List
-from typing import Tuple
-
-import jieba
-from pypinyin import lazy_pinyin
-from pypinyin import Style
-
-
-class ToneSandhi:
-    def __init__(self):
-        self.must_neural_tone_words = {
-            "麻烦",
-            "麻利",
-            "鸳鸯",
-            "高粱",
-            "骨头",
-            "骆驼",
-            "马虎",
-            "首饰",
-            "馒头",
-            "馄饨",
-            "风筝",
-            "难为",
-            "队伍",
-            "阔气",
-            "闺女",
-            "门道",
-            "锄头",
-            "铺盖",
-            "铃铛",
-            "铁匠",
-            "钥匙",
-            "里脊",
-            "里头",
-            "部分",
-            "那么",
-            "道士",
-            "造化",
-            "迷糊",
-            "连累",
-            "这么",
-            "这个",
-            "运气",
-            "过去",
-            "软和",
-            "转悠",
-            "踏实",
-            "跳蚤",
-            "跟头",
-            "趔趄",
-            "财主",
-            "豆腐",
-            "讲究",
-            "记性",
-            "记号",
-            "认识",
-            "规矩",
-            "见识",
-            "裁缝",
-            "补丁",
-            "衣裳",
-            "衣服",
-            "衙门",
-            "街坊",
-            "行李",
-            "行当",
-            "蛤蟆",
-            "蘑菇",
-            "薄荷",
-            "葫芦",
-            "葡萄",
-            "萝卜",
-            "荸荠",
-            "苗条",
-            "苗头",
-            "苍蝇",
-            "芝麻",
-            "舒服",
-            "舒坦",
-            "舌头",
-            "自在",
-            "膏药",
-            "脾气",
-            "脑袋",
-            "脊梁",
-            "能耐",
-            "胳膊",
-            "胭脂",
-            "胡萝",
-            "胡琴",
-            "胡同",
-            "聪明",
-            "耽误",
-            "耽搁",
-            "耷拉",
-            "耳朵",
-            "老爷",
-            "老实",
-            "老婆",
-            "老头",
-            "老太",
-            "翻腾",
-            "罗嗦",
-            "罐头",
-            "编辑",
-            "结实",
-            "红火",
-            "累赘",
-            "糨糊",
-            "糊涂",
-            "精神",
-            "粮食",
-            "簸箕",
-            "篱笆",
-            "算计",
-            "算盘",
-            "答应",
-            "笤帚",
-            "笑语",
-            "笑话",
-            "窟窿",
-            "窝囊",
-            "窗户",
-            "稳当",
-            "稀罕",
-            "称呼",
-            "秧歌",
-            "秀气",
-            "秀才",
-            "福气",
-            "祖宗",
-            "砚台",
-            "码头",
-            "石榴",
-            "石头",
-            "石匠",
-            "知识",
-            "眼睛",
-            "眯缝",
-            "眨巴",
-            "眉毛",
-            "相声",
-            "盘算",
-            "白净",
-            "痢疾",
-            "痛快",
-            "疟疾",
-            "疙瘩",
-            "疏忽",
-            "畜生",
-            "生意",
-            "甘蔗",
-            "琵琶",
-            "琢磨",
-            "琉璃",
-            "玻璃",
-            "玫瑰",
-            "玄乎",
-            "狐狸",
-            "状元",
-            "特务",
-            "牲口",
-            "牙碜",
-            "牌楼",
-            "爽快",
-            "爱人",
-            "热闹",
-            "烧饼",
-            "烟筒",
-            "烂糊",
-            "点心",
-            "炊帚",
-            "灯笼",
-            "火候",
-            "漂亮",
-            "滑溜",
-            "溜达",
-            "温和",
-            "清楚",
-            "消息",
-            "浪头",
-            "活泼",
-            "比方",
-            "正经",
-            "欺负",
-            "模糊",
-            "槟榔",
-            "棺材",
-            "棒槌",
-            "棉花",
-            "核桃",
-            "栅栏",
-            "柴火",
-            "架势",
-            "枕头",
-            "枇杷",
-            "机灵",
-            "本事",
-            "木头",
-            "木匠",
-            "朋友",
-            "月饼",
-            "月亮",
-            "暖和",
-            "明白",
-            "时候",
-            "新鲜",
-            "故事",
-            "收拾",
-            "收成",
-            "提防",
-            "挖苦",
-            "挑剔",
-            "指甲",
-            "指头",
-            "拾掇",
-            "拳头",
-            "拨弄",
-            "招牌",
-            "招呼",
-            "抬举",
-            "护士",
-            "折腾",
-            "扫帚",
-            "打量",
-            "打算",
-            "打点",
-            "打扮",
-            "打听",
-            "打发",
-            "扎实",
-            "扁担",
-            "戒指",
-            "懒得",
-            "意识",
-            "意思",
-            "情形",
-            "悟性",
-            "怪物",
-            "思量",
-            "怎么",
-            "念头",
-            "念叨",
-            "快活",
-            "忙活",
-            "志气",
-            "心思",
-            "得罪",
-            "张罗",
-            "弟兄",
-            "开通",
-            "应酬",
-            "庄稼",
-            "干事",
-            "帮手",
-            "帐篷",
-            "希罕",
-            "师父",
-            "师傅",
-            "巴结",
-            "巴掌",
-            "差事",
-            "工夫",
-            "岁数",
-            "屁股",
-            "尾巴",
-            "少爷",
-            "小气",
-            "小伙",
-            "将就",
-            "对头",
-            "对付",
-            "寡妇",
-            "家伙",
-            "客气",
-            "实在",
-            "官司",
-            "学问",
-            "学生",
-            "字号",
-            "嫁妆",
-            "媳妇",
-            "媒人",
-            "婆家",
-            "娘家",
-            "委屈",
-            "姑娘",
-            "姐夫",
-            "妯娌",
-            "妥当",
-            "妖精",
-            "奴才",
-            "女婿",
-            "头发",
-            "太阳",
-            "大爷",
-            "大方",
-            "大意",
-            "大夫",
-            "多少",
-            "多么",
-            "外甥",
-            "壮实",
-            "地道",
-            "地方",
-            "在乎",
-            "困难",
-            "嘴巴",
-            "嘱咐",
-            "嘟囔",
-            "嘀咕",
-            "喜欢",
-            "喇嘛",
-            "喇叭",
-            "商量",
-            "唾沫",
-            "哑巴",
-            "哈欠",
-            "哆嗦",
-            "咳嗽",
-            "和尚",
-            "告诉",
-            "告示",
-            "含糊",
-            "吓唬",
-            "后头",
-            "名字",
-            "名堂",
-            "合同",
-            "吆喝",
-            "叫唤",
-            "口袋",
-            "厚道",
-            "厉害",
-            "千斤",
-            "包袱",
-            "包涵",
-            "匀称",
-            "勤快",
-            "动静",
-            "动弹",
-            "功夫",
-            "力气",
-            "前头",
-            "刺猬",
-            "刺激",
-            "别扭",
-            "利落",
-            "利索",
-            "利害",
-            "分析",
-            "出息",
-            "凑合",
-            "凉快",
-            "冷战",
-            "冤枉",
-            "冒失",
-            "养活",
-            "关系",
-            "先生",
-            "兄弟",
-            "便宜",
-            "使唤",
-            "佩服",
-            "作坊",
-            "体面",
-            "位置",
-            "似的",
-            "伙计",
-            "休息",
-            "什么",
-            "人家",
-            "亲戚",
-            "亲家",
-            "交情",
-            "云彩",
-            "事情",
-            "买卖",
-            "主意",
-            "丫头",
-            "丧气",
-            "两口",
-            "东西",
-            "东家",
-            "世故",
-            "不由",
-            "不在",
-            "下水",
-            "下巴",
-            "上头",
-            "上司",
-            "丈夫",
-            "丈人",
-            "一辈",
-            "那个",
-            "菩萨",
-            "父亲",
-            "母亲",
-            "咕噜",
-            "邋遢",
-            "费用",
-            "冤家",
-            "甜头",
-            "介绍",
-            "荒唐",
-            "大人",
-            "泥鳅",
-            "幸福",
-            "熟悉",
-            "计划",
-            "扑腾",
-            "蜡烛",
-            "姥爷",
-            "照顾",
-            "喉咙",
-            "吉他",
-            "弄堂",
-            "蚂蚱",
-            "凤凰",
-            "拖沓",
-            "寒碜",
-            "糟蹋",
-            "倒腾",
-            "报复",
-            "逻辑",
-            "盘缠",
-            "喽啰",
-            "牢骚",
-            "咖喱",
-            "扫把",
-            "惦记",
-        }
-        self.must_not_neural_tone_words = {
-            "男子",
-            "女子",
-            "分子",
-            "原子",
-            "量子",
-            "莲子",
-            "石子",
-            "瓜子",
-            "电子",
-            "人人",
-            "虎虎",
-        }
-        self.punc = "：，；。？！“”‘’':,;.?!"
-
-    # the meaning of jieba pos tag: https://blog.csdn.net/weixin_44174352/article/details/113731041
-    # e.g.
-    # word: "家里"
-    # pos: "s"
-    # finals: ['ia1', 'i3']
-    def _neural_sandhi(self, word: str, pos: str, finals: List[str]) -> List[str]:
-        # reduplication words for n. and v. e.g. 奶奶, 试试, 旺旺
-        for j, item in enumerate(word):
-            if (
-                j - 1 >= 0
-                and item == word[j - 1]
-                and pos[0] in {"n", "v", "a"}
-                and word not in self.must_not_neural_tone_words
-            ):
-                finals[j] = finals[j][:-1] + "5"
-        ge_idx = word.find("个")
-        if len(word) >= 1 and word[-1] in "吧呢啊呐噻嘛吖嗨呐哦哒额滴哩哟喽啰耶喔诶":
-            finals[-1] = finals[-1][:-1] + "5"
-        elif len(word) >= 1 and word[-1] in "的地得":
-            finals[-1] = finals[-1][:-1] + "5"
-        # e.g. 走了, 看着, 去过
-        # elif len(word) == 1 and word in "了着过" and pos in {"ul", "uz", "ug"}:
-        #     finals[-1] = finals[-1][:-1] + "5"
-        elif (
-            len(word) > 1
-            and word[-1] in "们子"
-            and pos in {"r", "n"}
-            and word not in self.must_not_neural_tone_words
-        ):
-            finals[-1] = finals[-1][:-1] + "5"
-        # e.g. 桌上, 地下, 家里
-        elif len(word) > 1 and word[-1] in "上下里" and pos in {"s", "l", "f"}:
-            finals[-1] = finals[-1][:-1] + "5"
-        # e.g. 上来, 下去
-        elif len(word) > 1 and word[-1] in "来去" and word[-2] in "上下进出回过起开":
-            finals[-1] = finals[-1][:-1] + "5"
-        # 个做量词
-        elif (
-            ge_idx >= 1
-            and (word[ge_idx - 1].isnumeric() or word[ge_idx - 1] in "几有两半多各整每做是")
-        ) or word == "个":
-            finals[ge_idx] = finals[ge_idx][:-1] + "5"
-        else:
-            if (
-                word in self.must_neural_tone_words
-                or word[-2:] in self.must_neural_tone_words
-            ):
-                finals[-1] = finals[-1][:-1] + "5"
-
-        word_list = self._split_word(word)
-        finals_list = [finals[: len(word_list[0])], finals[len(word_list[0]) :]]
-        for i, word in enumerate(word_list):
-            # conventional neural in Chinese
-            if (
-                word in self.must_neural_tone_words
-                or word[-2:] in self.must_neural_tone_words
-            ):
-                finals_list[i][-1] = finals_list[i][-1][:-1] + "5"
-        finals = sum(finals_list, [])
-        return finals
-
-    def _bu_sandhi(self, word: str, finals: List[str]) -> List[str]:
-        # e.g. 看不懂
-        if len(word) == 3 and word[1] == "不":
-            finals[1] = finals[1][:-1] + "5"
-        else:
-            for i, char in enumerate(word):
-                # "不" before tone4 should be bu2, e.g. 不怕
-                if char == "不" and i + 1 < len(word) and finals[i + 1][-1] == "4":
-                    finals[i] = finals[i][:-1] + "2"
-        return finals
-
-    def _yi_sandhi(self, word: str, finals: List[str]) -> List[str]:
-        # "一" in number sequences, e.g. 一零零, 二一零
-        if word.find("一") != -1 and all(
-            [item.isnumeric() for item in word if item != "一"]
-        ):
-            return finals
-        # "一" between reduplication words should be yi5, e.g. 看一看
-        elif len(word) == 3 and word[1] == "一" and word[0] == word[-1]:
-            finals[1] = finals[1][:-1] + "5"
-        # when "一" is ordinal word, it should be yi1
-        elif word.startswith("第一"):
-            finals[1] = finals[1][:-1] + "1"
-        else:
-            for i, char in enumerate(word):
-                if char == "一" and i + 1 < len(word):
-                    # "一" before tone4 should be yi2, e.g. 一段
-                    if finals[i + 1][-1] == "4":
-                        finals[i] = finals[i][:-1] + "2"
-                    # "一" before non-tone4 should be yi4, e.g. 一天
-                    else:
-                        # "一" 后面如果是标点，还读一声
-                        if word[i + 1] not in self.punc:
-                            finals[i] = finals[i][:-1] + "4"
-        return finals
-
-    def _split_word(self, word: str) -> List[str]:
-        word_list = jieba.cut_for_search(word)
-        word_list = sorted(word_list, key=lambda i: len(i), reverse=False)
-        first_subword = word_list[0]
-        first_begin_idx = word.find(first_subword)
-        if first_begin_idx == 0:
-            second_subword = word[len(first_subword) :]
-            new_word_list = [first_subword, second_subword]
-        else:
-            second_subword = word[: -len(first_subword)]
-            new_word_list = [second_subword, first_subword]
-        return new_word_list
-
-    def _three_sandhi(self, word: str, finals: List[str]) -> List[str]:
-        if len(word) == 2 and self._all_tone_three(finals):
-            finals[0] = finals[0][:-1] + "2"
-        elif len(word) == 3:
-            word_list = self._split_word(word)
-            if self._all_tone_three(finals):
-                #  disyllabic + monosyllabic, e.g. 蒙古/包
-                if len(word_list[0]) == 2:
-                    finals[0] = finals[0][:-1] + "2"
-                    finals[1] = finals[1][:-1] + "2"
-                #  monosyllabic + disyllabic, e.g. 纸/老虎
-                elif len(word_list[0]) == 1:
-                    finals[1] = finals[1][:-1] + "2"
-            else:
-                finals_list = [finals[: len(word_list[0])], finals[len(word_list[0]) :]]
-                if len(finals_list) == 2:
-                    for i, sub in enumerate(finals_list):
-                        # e.g. 所有/人
-                        if self._all_tone_three(sub) and len(sub) == 2:
-                            finals_list[i][0] = finals_list[i][0][:-1] + "2"
-                        # e.g. 好/喜欢
-                        elif (
-                            i == 1
-                            and not self._all_tone_three(sub)
-                            and finals_list[i][0][-1] == "3"
-                            and finals_list[0][-1][-1] == "3"
-                        ):
-                            finals_list[0][-1] = finals_list[0][-1][:-1] + "2"
-                        finals = sum(finals_list, [])
-        # split idiom into two words who's length is 2
-        elif len(word) == 4:
-            finals_list = [finals[:2], finals[2:]]
-            finals = []
-            for sub in finals_list:
-                if self._all_tone_three(sub):
-                    sub[0] = sub[0][:-1] + "2"
-                finals += sub
-
-        return finals
-
-    def _all_tone_three(self, finals: List[str]) -> bool:
-        return all(x[-1] == "3" for x in finals)
-
-    # merge "不" and the word behind it
-    # if don't merge, "不" sometimes appears alone according to jieba, which may occur sandhi error
-    def _merge_bu(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
-        new_seg = []
-        last_word = ""
-        for word, pos in seg:
-            if last_word == "不":
-                word = last_word + word
-            if word != "不":
-                new_seg.append((word, pos))
-            last_word = word[:]
-        if last_word == "不":
-            new_seg.append((last_word, "d"))
-            last_word = ""
-        return new_seg
-
-    # function 1: merge "一" and reduplication words in it's left and right, e.g. "听","一","听" ->"听一听"
-    # function 2: merge single  "一" and the word behind it
-    # if don't merge, "一" sometimes appears alone according to jieba, which may occur sandhi error
-    # e.g.
-    # input seg: [('听', 'v'), ('一', 'm'), ('听', 'v')]
-    # output seg: [['听一听', 'v']]
-    def _merge_yi(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
-        new_seg = [] * len(seg)
-        # function 1
-        i = 0
-        while i < len(seg):
-            word, pos = seg[i]
-            if (
-                i - 1 >= 0
-                and word == "一"
-                and i + 1 < len(seg)
-                and seg[i - 1][0] == seg[i + 1][0]
-                and seg[i - 1][1] == "v"
-            ):
-                new_seg[i - 1][0] = new_seg[i - 1][0] + "一" + new_seg[i - 1][0]
-                i += 2
-            else:
-                if (
-                    i - 2 >= 0
-                    and seg[i - 1][0] == "一"
-                    and seg[i - 2][0] == word
-                    and pos == "v"
-                ):
-                    continue
-                else:
-                    new_seg.append([word, pos])
-                i += 1
-        seg = [i for i in new_seg if len(i) > 0]
-        new_seg = []
-        # function 2
-        for i, (word, pos) in enumerate(seg):
-            if new_seg and new_seg[-1][0] == "一":
-                new_seg[-1][0] = new_seg[-1][0] + word
-            else:
-                new_seg.append([word, pos])
-        return new_seg
-
-    # the first and the second words are all_tone_three
-    def _merge_continuous_three_tones(
-        self, seg: List[Tuple[str, str]]
-    ) -> List[Tuple[str, str]]:
-        new_seg = []
-        sub_finals_list = [
-            lazy_pinyin(word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
-            for (word, pos) in seg
-        ]
-        assert len(sub_finals_list) == len(seg)
-        merge_last = [False] * len(seg)
-        for i, (word, pos) in enumerate(seg):
-            if (
-                i - 1 >= 0
-                and self._all_tone_three(sub_finals_list[i - 1])
-                and self._all_tone_three(sub_finals_list[i])
-                and not merge_last[i - 1]
-            ):
-                # if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi
-                if (
-                    not self._is_reduplication(seg[i - 1][0])
-                    and len(seg[i - 1][0]) + len(seg[i][0]) <= 3
-                ):
-                    new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
-                    merge_last[i] = True
-                else:
-                    new_seg.append([word, pos])
-            else:
-                new_seg.append([word, pos])
-
-        return new_seg
-
-    def _is_reduplication(self, word: str) -> bool:
-        return len(word) == 2 and word[0] == word[1]
-
-    # the last char of first word and the first char of second word is tone_three
-    def _merge_continuous_three_tones_2(
-        self, seg: List[Tuple[str, str]]
-    ) -> List[Tuple[str, str]]:
-        new_seg = []
-        sub_finals_list = [
-            lazy_pinyin(word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
-            for (word, pos) in seg
-        ]
-        assert len(sub_finals_list) == len(seg)
-        merge_last = [False] * len(seg)
-        for i, (word, pos) in enumerate(seg):
-            if (
-                i - 1 >= 0
-                and sub_finals_list[i - 1][-1][-1] == "3"
-                and sub_finals_list[i][0][-1] == "3"
-                and not merge_last[i - 1]
-            ):
-                # if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi
-                if (
-                    not self._is_reduplication(seg[i - 1][0])
-                    and len(seg[i - 1][0]) + len(seg[i][0]) <= 3
-                ):
-                    new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
-                    merge_last[i] = True
-                else:
-                    new_seg.append([word, pos])
-            else:
-                new_seg.append([word, pos])
-        return new_seg
-
-    def _merge_er(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
-        new_seg = []
-        for i, (word, pos) in enumerate(seg):
-            if i - 1 >= 0 and word == "儿" and seg[i - 1][0] != "#":
-                new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
-            else:
-                new_seg.append([word, pos])
-        return new_seg
-
-    def _merge_reduplication(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
-        new_seg = []
-        for i, (word, pos) in enumerate(seg):
-            if new_seg and word == new_seg[-1][0]:
-                new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
-            else:
-                new_seg.append([word, pos])
-        return new_seg
-
-    def pre_merge_for_modify(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
-        seg = self._merge_bu(seg)
-        try:
-            seg = self._merge_yi(seg)
-        except:
-            print("_merge_yi failed")
-        seg = self._merge_reduplication(seg)
-        seg = self._merge_continuous_three_tones(seg)
-        seg = self._merge_continuous_three_tones_2(seg)
-        seg = self._merge_er(seg)
-        return seg
-
-    def modified_tone(self, word: str, pos: str, finals: List[str]) -> List[str]:
-        finals = self._bu_sandhi(word, finals)
-        finals = self._yi_sandhi(word, finals)
-        finals = self._neural_sandhi(word, pos, finals)
-        finals = self._three_sandhi(word, finals)
-        return finals