import tempfile import gradio as gr import soundfile as sf import torchaudio from cached_path import cached_path from omegaconf import OmegaConf from ipa.ipa import get_ipa, parse_ipa try: import spaces USING_SPACES = True except ImportError: USING_SPACES = False from f5_tts.infer.utils_infer import ( infer_process, load_model, load_vocoder, preprocess_ref_audio_text, remove_silence_for_generated_wav, save_spectrogram, ) from f5_tts.model import DiT def gpu_decorator(func): if USING_SPACES: return spaces.GPU(func) else: return func vocoder = load_vocoder() def load_f5tts(ckpt_path, vocab_path): ckpt_path = str(cached_path(ckpt_path)) F5TTS_model_cfg = dict( dim=1024, depth=22, heads=16, ff_mult=2, text_dim=512, conv_layers=4 ) vocab_path = str(cached_path(vocab_path)) return load_model(DiT, F5TTS_model_cfg, ckpt_path, vocab_file=vocab_path) OmegaConf.register_new_resolver("load_f5tts", load_f5tts) models_config = OmegaConf.to_object(OmegaConf.load("configs/models.yaml")) dialects = OmegaConf.to_object(OmegaConf.load("configs/dialects.yaml")) DEFAULT_MODEL_ID = list(models_config.keys())[0] DEFAULT_DIALECT = list(dialects.values())[0] @gpu_decorator def infer( ref_audio_orig, ref_text, gen_text, model, remove_silence, cross_fade_duration=0.15, nfe_step=32, fix_duration=1, show_info=gr.Info, ): if not ref_audio_orig: gr.Warning("Please provide reference audio.") return gr.update(), gr.update(), ref_text if not gen_text.strip(): gr.Warning("Please enter text to generate.") return gr.update(), gr.update(), ref_text ref_audio, ref_text = preprocess_ref_audio_text( ref_audio_orig, ref_text, show_info=show_info ) final_wave, final_sample_rate, combined_spectrogram = infer_process( ref_audio, ref_text, gen_text, model, vocoder, cross_fade_duration=cross_fade_duration, nfe_step=nfe_step, fix_duration=fix_duration, show_info=show_info, progress=gr.Progress(), ) # Remove silence if remove_silence: with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as f: sf.write(f.name, final_wave, final_sample_rate) remove_silence_for_generated_wav(f.name) final_wave, _ = torchaudio.load(f.name) final_wave = final_wave.squeeze().cpu().numpy() print(f"Final wave duration: {final_wave.shape[0] / final_sample_rate:.2f}s") # Save the spectrogram with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp_spectrogram: spectrogram_path = tmp_spectrogram.name save_spectrogram(combined_spectrogram, spectrogram_path) return (final_sample_rate, final_wave), spectrogram_path demo = gr.Blocks( title="臺灣客語語音生成系統", css="@import url(https://tauhu.tw/tauhu-oo.css);", theme=gr.themes.Default( font=( "tauhu-oo", gr.themes.GoogleFont("Source Sans Pro"), "ui-sans-serif", "system-ui", "sans-serif", ) ), ) with demo: gr.Markdown( """ # 臺灣客語語音合成系統 ### Taiwanese Hakka Text-to-Speech System ### 研發團隊 - **[李鴻欣 Hung-Shin Lee](mailto:hungshinlee@gmail.com)（[聯和科創](https://www.104.com.tw/company/1a2x6bmu75)）** - **[陳力瑋 Li-Wei Chen](mailto:wayne900619@gmail.com)（[聯和科創](https://www.104.com.tw/company/1a2x6bmu75)）** ### 合作單位 - **[國立聯合大學智慧客家實驗室](https://www.gohakka.org)** """ ) with gr.Row(): with gr.Column(): model_drop_down = gr.Dropdown( models_config.keys(), value=DEFAULT_MODEL_ID, label="模型", ) ref_audio_input = gr.Audio( type="filepath", waveform_options=gr.WaveformOptions( sample_rate=24000, ), label="Reference Audio", ) ref_text_input = gr.Textbox( value="", label="Reference Text", ) ref_dialect_radio = gr.Radio( choices=[(k, v) for k, v in dialects.items()], value=DEFAULT_DIALECT, label="ref 腔調", interactive=len(dialects.keys()) > 1, ) gen_text_input = gr.Textbox( label="Text to Generate", value="", ) dialect_radio = gr.Radio( choices=[(k, v) for k, v in dialects.items()], value=DEFAULT_DIALECT, label="腔調", interactive=len(dialects.keys()) > 1, ) generate_btn = gr.Button("Synthesize", variant="primary") with gr.Accordion("Advanced Settings", open=False): remove_silence = gr.Checkbox( label="Remove Silences", info="The model tends to produce silences, especially on longer audio. We can manually remove silences if needed. Note that this is an experimental feature and may produce strange results. This will also increase generation time.", value=False, ) speed_slider = gr.Slider( label="Speed", minimum=0.3, maximum=2.0, value=1.0, step=0.1, info="語速（越小越慢）", ) nfe_slider = gr.Slider( label="NFE Steps", minimum=4, maximum=64, value=32, step=2, info="Set the number of denoising steps.", ) cross_fade_duration_slider = gr.Slider( label="Cross-Fade Duration (s)", minimum=0.0, maximum=1.0, value=0.15, step=0.01, info="Set the duration of the cross-fade between audio clips.", ) with gr.Column(): audio_output = gr.Audio(label="Synthesized Audio") spectrogram_output = gr.Image(label="Spectrogram") @gpu_decorator def basic_tts( model_drop_down: str, ref_audio_input: str, ref_text_input: str, ref_dialect_radio: str, gen_text_input: str, dialect_radio: str, remove_silence: bool, cross_fade_duration_slider: float, nfe_slider: int, speed_slider: float, ): ref_audio_info = torchaudio.info(ref_audio_input) ref_duration = ref_audio_info.num_frames / ref_audio_info.sample_rate target_duration = ( ref_duration * len(gen_text_input.replace(" ", "")) / len(ref_text_input.replace(" ", "")) / speed_slider ) print(f"Reference duration: {ref_duration}") print(f"Target duration: {target_duration}") if len(ref_text_input) == 0: raise gr.Error("請勿輸入空字串。") words, ipa, pinyin, missing_words = get_ipa( ref_text_input, dialect=ref_dialect_radio ) if len(missing_words) > 0: raise gr.Error( f"參考句子中的[{','.join(missing_words)}]目前無法轉成 ipa。請嘗試其他句子。" ) ref_text_input = parse_ipa(ipa) if len(gen_text_input) == 0: raise gr.Error("請勿輸入空字串。") words, ipa, pinyin, missing_words = get_ipa( gen_text_input, dialect=dialect_radio ) if len(missing_words) > 0: raise gr.Error( f"生成句子中的[{','.join(missing_words)}]目前無法轉成 ipa。請嘗試其他句子。" ) gen_text_input = parse_ipa(ipa) audio_out, spectrogram_path = infer( ref_audio_input, ref_text_input, gen_text_input, models_config[model_drop_down], remove_silence, cross_fade_duration=cross_fade_duration_slider, nfe_step=nfe_slider, fix_duration=ref_duration + target_duration, ) return audio_out, spectrogram_path generate_btn.click( basic_tts, inputs=[ model_drop_down, ref_audio_input, ref_text_input, ref_dialect_radio, gen_text_input, dialect_radio, remove_silence, cross_fade_duration_slider, nfe_slider, speed_slider, ], outputs=[audio_output, spectrogram_output], ) gr.Examples( [ [ "./ref_wav/0000001_0.15-0.93.wav", "恁早", "sixian", "食飯愛正經食，正毋會食到半出半入", "sixian", ], [ "./ref_wav/0000002_0.15-2.73.wav", "你今晡日著到恁派頭", "sixian", "食飯愛正經食，正毋會食到半出半入", "sixian", ], [ "./ref_wav/0000002_0.15-2.73.wav", "你今晡日著到恁派頭", "sixian", "歸條路吊等長長个花燈，祈求風調雨順，歸屋下人个心願，親像花燈下燒暖个光華", "sixian", ], # [ # "預設語者", # "戴君儒", # "hailu", # "男女平等个時代，平平做得受教育", # ], # [ # "預設語者", # "宋涵葳", # "dapu", # "客家山城乜跈緊鬧熱䟘來咧", # ], # [ # "預設語者", # "江芮敏", # "raoping", # "頭擺匱人，戴个毋係菅草屋，个創商品哦", # ], # [ # "預設語者", # "洪藝晅", # "zhaoan", # "歇熱个時務，阿松歸屋下轉去在客莊个老屋", # ], # [ # "預設語者", # "江芮敏", # "nansixian", # "在𠊎讀小學一年生个時節，阿爸輒常用自轉車載𠊎去學校讀書", # ], ], label="範例", inputs=[ ref_audio_input, ref_text_input, ref_dialect_radio, gen_text_input, dialect_radio, ], ) demo.launch()