Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,33 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Musetalk/data/video/man_musev.mp4 filter=lfs diff=lfs merge=lfs -text
+Musetalk/data/video/monalisa_musev.mp4 filter=lfs diff=lfs merge=lfs -text
+Musetalk/data/video/seaside4_musev.mp4 filter=lfs diff=lfs merge=lfs -text
+Musetalk/data/video/sit_musev.mp4 filter=lfs diff=lfs merge=lfs -text
+Musetalk/data/video/sun_musev.mp4 filter=lfs diff=lfs merge=lfs -text
+Musetalk/data/video/yongen_musev.mp4 filter=lfs diff=lfs merge=lfs -text
+NeRF/gridencoder/build/lib.linux-x86_64-3.10/_gridencoder.cpython-310-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+NeRF/gridencoder/build/temp.linux-x86_64-3.10/root/Linly-Talker/NeRF/gridencoder/src/gridencoder.o filter=lfs diff=lfs merge=lfs -text
+docs/WebUI.png filter=lfs diff=lfs merge=lfs -text
+examples/source_image/art_16.png filter=lfs diff=lfs merge=lfs -text
+examples/source_image/art_17.png filter=lfs diff=lfs merge=lfs -text
+examples/source_image/art_3.png filter=lfs diff=lfs merge=lfs -text
+examples/source_image/art_4.png filter=lfs diff=lfs merge=lfs -text
+examples/source_image/art_5.png filter=lfs diff=lfs merge=lfs -text
+examples/source_image/art_8.png filter=lfs diff=lfs merge=lfs -text
+examples/source_image/art_9.png filter=lfs diff=lfs merge=lfs -text
+inputs/boy.png filter=lfs diff=lfs merge=lfs -text
+pytorch3d/.github/bundle_adjust.gif filter=lfs diff=lfs merge=lfs -text
+pytorch3d/.github/camera_position_teapot.gif filter=lfs diff=lfs merge=lfs -text
+pytorch3d/.github/fit_nerf.gif filter=lfs diff=lfs merge=lfs -text
+pytorch3d/.github/fit_textured_volume.gif filter=lfs diff=lfs merge=lfs -text
+pytorch3d/.github/implicitron_config.gif filter=lfs diff=lfs merge=lfs -text
+pytorch3d/.github/nerf_project_logo.gif filter=lfs diff=lfs merge=lfs -text
+pytorch3d/build/lib.linux-x86_64-3.10/pytorch3d/_C.cpython-310-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+pytorch3d/build/temp.linux-x86_64-3.10/.ninja_deps filter=lfs diff=lfs merge=lfs -text
+pytorch3d/build/temp.linux-x86_64-3.10/root/Linly-Talker/pytorch3d/pytorch3d/csrc/knn/knn.o filter=lfs diff=lfs merge=lfs -text
+pytorch3d/build/temp.linux-x86_64-3.10/root/Linly-Talker/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.construct.gpu.o filter=lfs diff=lfs merge=lfs -text
+pytorch3d/build/temp.linux-x86_64-3.10/root/Linly-Talker/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.forward.gpu.o filter=lfs diff=lfs merge=lfs -text
+pytorch3d/docs/notes/assets/batch_modes.gif filter=lfs diff=lfs merge=lfs -text
+pytorch3d/docs/notes/assets/meshrcnn.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,15 @@
+.DS_Store
+checkpoints/
+gfpgan/
+__pycache__/
+*.pyc
+Linly-AI
+Qwen
+checkpoints
+temp
+*.wav
+*.vtt
+*.srt
+results/example_answer.mp4
+request-Linly-api.py
+results

.gitmodules

@@ -0,0 +1,7 @@
+[submodule "MuseV"]
+	path = MuseV
+	url = https://github.com/TMElyralab/MuseV.git
+
+[submodule "ChatTTS"]
+	path = ChatTTS
+	url = https://github.com/2noise/ChatTTS.git

.ipynb_checkpoints/webui-checkpoint.py

@@ -0,0 +1,1364 @@
+import os
+import random 
+import gradio as gr
+import time
+import torch
+import gc
+import warnings
+warnings.filterwarnings('ignore')
+from zhconv import convert
+from LLM import LLM
+from TTS import EdgeTTS
+from src.cost_time import calculate_time
+
+from configs import *
+os.environ["GRADIO_TEMP_DIR"]= './temp'
+os.environ["WEBUI"] = "true"
+def get_title(title = 'Linly 智能对话系统 (Linly-Talker)'):
+    description = f"""
+    <p style="text-align: center; font-weight: bold;">
+        <span style="font-size: 28px;">{title}</span>
+        <br>
+        <span style="font-size: 18px;" id="paper-info">
+            [<a href="https://zhuanlan.zhihu.com/p/671006998" target="_blank">知乎</a>]
+            [<a href="https://www.bilibili.com/video/BV1rN4y1a76x/" target="_blank">bilibili</a>]
+            [<a href="https://github.com/Kedreamix/Linly-Talker" target="_blank">GitHub</a>]
+            [<a herf="https://kedreamix.github.io/" target="_blank">个人主页</a>]
+        </span>
+        <br> 
+        <span>Linly-Talker是一款创新的数字人对话系统，它融合了最新的人工智能技术，包括大型语言模型（LLM）🤖、自动语音识别（ASR）🎙️、文本到语音转换（TTS）🗣️和语音克隆技术🎤。</span>
+    </p>
+    """
+    return description
+
+
+# 设置默认system
+default_system = '你是一个很有帮助的助手'
+# 设置默认的prompt
+prefix_prompt = '''请用少于25个字回答以下问题\n\n'''
+
+edgetts = EdgeTTS()
+
+# 设定默认参数值，可修改
+blink_every = True
+size_of_image = 256
+preprocess_type = 'crop'
+facerender = 'facevid2vid'
+enhancer = False
+is_still_mode = False
+exp_weight = 1
+use_ref_video = False
+ref_video = None
+ref_info = 'pose'
+use_idle_mode = False
+length_of_audio = 5
+
+@calculate_time
+def Asr(audio):
+    try:
+        question = asr.transcribe(audio)
+        question = convert(question, 'zh-cn')
+    except Exception as e:
+        print("ASR Error: ", e)
+        question = 'Gradio存在一些bug，麦克风模式有时候可能音频还未传入，请重新点击一下语音识别即可'
+        gr.Warning(question)
+    return question
+
+def clear_memory():
+    """
+    清理PyTorch的显存和系统内存缓存。
+    """
+    # 1. 清理缓存的变量
+    gc.collect()  # 触发Python垃圾回收
+    torch.cuda.empty_cache()  # 清理PyTorch的显存缓存
+    torch.cuda.ipc_collect()  # 清理PyTorch的跨进程通信缓存
+    
+    # 2. 打印显存使用情况（可选）
+    print(f"Memory allocated: {torch.cuda.memory_allocated() / (1024 ** 2):.2f} MB")
+    print(f"Max memory allocated: {torch.cuda.max_memory_allocated() / (1024 ** 2):.2f} MB")
+    print(f"Cached memory: {torch.cuda.memory_reserved() / (1024 ** 2):.2f} MB")
+    print(f"Max cached memory: {torch.cuda.max_memory_reserved() / (1024 ** 2):.2f} MB")
+
+@calculate_time
+def TTS_response(text, 
+                 voice, rate, volume, pitch,
+                 am, voc, lang, male,
+                 inp_ref, prompt_text, prompt_language, text_language, how_to_cut, 
+                 question_audio, question, use_mic_voice,
+                 tts_method = 'PaddleTTS', save_path = 'answer.wav'):
+    # print(text, voice, rate, volume, pitch, am, voc, lang, male, tts_method, save_path)
+    if tts_method == 'Edge-TTS':
+        if not edgetts.network:
+            gr.Warning("请检查网络或者使用其他模型，例如PaddleTTS") 
+            return None, None
+        try:
+            edgetts.predict(text, voice, rate, volume, pitch , 'answer.wav', 'answer.vtt')
+        except:
+            os.system(f'edge-tts --text "{text}" --voice {voice} --write-media answer.wav --write-subtitles answer.vtt')
+        return 'answer.wav', 'answer.vtt'
+    elif tts_method == 'PaddleTTS':
+        tts.predict(text, am, voc, lang = lang, male=male, save_path = save_path)
+        return save_path, None
+    elif tts_method == 'GPT-SoVITS克隆声音':
+        if use_mic_voice:
+            try:
+                vits.predict(ref_wav_path = question_audio,
+                                prompt_text = question,
+                                prompt_language = "中文",
+                                text = text, # 回答
+                                text_language = "中文",
+                                how_to_cut = "凑四句一切",
+                                save_path = 'answer.wav')
+                return 'answer.wav', None
+            except Exception as e:
+                gr.Warning("无克隆环境或者无克隆模型权重，无法克隆声音", e)
+                return None, None
+        else:
+            try:
+                vits.predict(ref_wav_path = inp_ref,
+                                prompt_text = prompt_text,
+                                prompt_language = prompt_language,
+                                text = text, # 回答
+                                text_language = text_language,
+                                how_to_cut = how_to_cut,
+                                save_path = 'answer.wav')
+                return 'answer.wav', None
+            except Exception as e:
+                gr.Warning("无克隆环境或者无克隆模型权重，无法克隆声音", e)
+                return None, None
+    return None, None
+@calculate_time
+def LLM_response(question_audio, question, 
+                 voice = 'zh-CN-XiaoxiaoNeural', rate = 0, volume = 0, pitch = 0,
+                 am='fastspeech2', voc='pwgan',lang='zh', male=False, 
+                 inp_ref = None, prompt_text = "", prompt_language = "", text_language = "", how_to_cut = "", use_mic_voice = False,
+                 tts_method = 'Edge-TTS'):
+    if len(question) == 0:
+        gr.Warning("请输入问题")
+        return None, None, None
+    answer = llm.generate(question, default_system)
+    print(answer)
+    driven_audio, driven_vtt = TTS_response(answer, voice, rate, volume, pitch, 
+                 am, voc, lang, male, 
+                 inp_ref, prompt_text, prompt_language, text_language, how_to_cut, question_audio, question, use_mic_voice,
+                 tts_method)
+    return driven_audio, driven_vtt, answer
+
+@calculate_time
+def Talker_response(question_audio = None, method = 'SadTalker', text = '',
+                    voice = 'zh-CN-XiaoxiaoNeural', rate = 0, volume = 100, pitch = 0, 
+                    am = 'fastspeech2', voc = 'pwgan', lang = 'zh', male = False, 
+                    inp_ref = None, prompt_text = "", prompt_language = "", text_language = "", how_to_cut = "", use_mic_voice = False,
+                    tts_method = 'Edge-TTS',batch_size = 2, character = '女性角色', 
+                    progress=gr.Progress(track_tqdm=True)):
+    default_voice = None
+    if character == '女性角色':
+        # 女性角色
+        source_image, pic_path = r'inputs/girl.png', r'inputs/girl.png'
+        crop_pic_path = "./inputs/first_frame_dir_girl/girl.png"
+        first_coeff_path = "./inputs/first_frame_dir_girl/girl.mat"
+        crop_info = ((403, 403), (19, 30, 502, 513), [40.05956541381802, 40.17324339233366, 443.7892505041507, 443.9029284826663])
+        default_voice = 'zh-CN-XiaoxiaoNeural'
+    elif character == '男性角色':
+        # 男性角色
+        source_image = r'./inputs/boy.png'
+        pic_path = "./inputs/boy.png"
+        crop_pic_path = "./inputs/first_frame_dir_boy/boy.png"
+        first_coeff_path = "./inputs/first_frame_dir_boy/boy.mat"
+        crop_info = ((876, 747), (0, 0, 886, 838), [10.382158280494476, 0, 886, 747.7078990925525])
+        default_voice = 'zh-CN-YunyangNeural'
+    else:
+        gr.Warning('未知角色')
+        return None
+    
+    voice = default_voice if not voice else voice
+    
+    if not voice:
+        gr.Warning('请选择声音')
+    
+    driven_audio, driven_vtt, _ = LLM_response(question_audio, text, 
+                                               voice, rate, volume, pitch, 
+                                               am, voc, lang, male, 
+                                               inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
+                                               tts_method)
+    if driven_audio is None:
+        gr.Warning("音频没有正常生成，请检查TTS是否正确")
+        return None
+    if method == 'SadTalker':
+        pose_style = random.randint(0, 45)
+        video = talker.test(pic_path,
+                        crop_pic_path,
+                        first_coeff_path,
+                        crop_info,
+                        source_image,
+                        driven_audio,
+                        preprocess_type,
+                        is_still_mode,
+                        enhancer,
+                        batch_size,                            
+                        size_of_image,
+                        pose_style,
+                        facerender,
+                        exp_weight,
+                        use_ref_video,
+                        ref_video,
+                        ref_info,
+                        use_idle_mode,
+                        length_of_audio,
+                        blink_every,
+                        fps=20)
+    elif method == 'Wav2Lip':
+        video = talker.predict(crop_pic_path, driven_audio, batch_size, enhancer)
+    elif method == 'NeRFTalk':
+        video = talker.predict(driven_audio)
+    else:
+        gr.Warning("不支持的方法：" + method)
+        return None
+    if driven_vtt:
+        return video, driven_vtt
+    else:
+        return video
+
+@calculate_time
+def Talker_response_img(question_audio, method, text, voice, rate, volume, pitch, 
+                        am, voc, lang, male, 
+                        inp_ref , prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
+                        tts_method,
+                        source_image,
+                        preprocess_type, 
+                        is_still_mode,
+                        enhancer,
+                        batch_size,                            
+                        size_of_image,
+                        pose_style,
+                        facerender,
+                        exp_weight,
+                        blink_every,
+                        fps, progress=gr.Progress(track_tqdm=True)
+                    ):
+    if enhancer:
+        gr.Warning("记得请先安装GFPGAN库，pip install gfpgan, 已安装可忽略")
+    if not voice:
+        gr.Warning("请先选择声音")
+    driven_audio, driven_vtt, _ = LLM_response(question_audio, text, voice, rate, volume, pitch, 
+                                               am, voc, lang, male, 
+                                               inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
+                                               tts_method = tts_method)
+    if driven_audio is None:
+        gr.Warning("音频没有正常生成，请检查TTS是否正确")
+        return None
+    if method == 'SadTalker':
+        video = talker.test2(source_image,
+                        driven_audio,
+                        preprocess_type,
+                        is_still_mode,
+                        enhancer,
+                        batch_size,                            
+                        size_of_image,
+                        pose_style,
+                        facerender,
+                        exp_weight,
+                        use_ref_video,
+                        ref_video,
+                        ref_info,
+                        use_idle_mode,
+                        length_of_audio,
+                        blink_every,
+                        fps=fps)
+    elif method == 'Wav2Lip':
+        video = talker.predict(source_image, driven_audio, batch_size)
+    elif method == 'NeRFTalk':
+        video = talker.predict(driven_audio)
+    else:
+        return None
+    if driven_vtt:
+        return video, driven_vtt
+    else:
+        return video
+
+@calculate_time
+def Talker_Say(preprocess_type, 
+                        is_still_mode,
+                        enhancer,
+                        batch_size,                            
+                        size_of_image,
+                        pose_style,
+                        facerender,
+                        exp_weight,
+                        blink_every,
+                        fps,source_image = None, source_video = None, question_audio = None, method = 'SadTalker', text = '', 
+                    voice = 'zh-CN-XiaoxiaoNeural', rate = 0, volume = 100, pitch = 0, 
+                    am = 'fastspeech2', voc = 'pwgan', lang = 'zh', male = False, 
+                    inp_ref = None, prompt_text = "", prompt_language = "", text_language = "", how_to_cut = "", use_mic_voice = False,
+                    tts_method = 'Edge-TTS', character = '女性角色',
+                    progress=gr.Progress(track_tqdm=True)):
+    if source_video:
+        source_image = source_video
+    default_voice = None
+    
+    voice = default_voice if not voice else voice
+    
+    if not voice:
+        gr.Warning('请选择声音')
+    
+    driven_audio, driven_vtt = TTS_response(text, voice, rate, volume, pitch, 
+                 am, voc, lang, male, 
+                 inp_ref, prompt_text, prompt_language, text_language, how_to_cut, question_audio, text, use_mic_voice,
+                 tts_method)
+    if driven_audio is None:
+        gr.Warning("音频没有正常生成，请检查TTS是否正确")
+        return None
+    if method == 'SadTalker':
+        pose_style = random.randint(0, 45)
+        video = talker.test2(source_image,
+                        driven_audio,
+                        preprocess_type,
+                        is_still_mode,
+                        enhancer,
+                        batch_size,                            
+                        size_of_image,
+                        pose_style,
+                        facerender,
+                        exp_weight,
+                        use_ref_video,
+                        ref_video,
+                        ref_info,
+                        use_idle_mode,
+                        length_of_audio,
+                        blink_every,
+                        fps=fps)
+    elif method == 'Wav2Lip':
+        video = talker.predict(source_image, driven_audio, batch_size, enhancer)
+    elif method == 'NeRFTalk':
+        video = talker.predict(driven_audio)
+    else:
+        gr.Warning("不支持的方法：" + method)
+        return None
+    if driven_vtt:
+        return video, driven_vtt
+    else:
+        return video
+
+def chat_response(system, message, history):
+    # response = llm.generate(message)
+    response, history = llm.chat(system, message, history)
+    print(history)
+    # 流式输出
+    for i in range(len(response)):
+        time.sleep(0.01)
+        yield "", history[:-1] + [(message, response[:i+1])]
+    return "", history
+
+def modify_system_session(system: str) -> str:
+    if system is None or len(system) == 0:
+        system = default_system
+    llm.clear_history()
+    return system, system, []
+
+def clear_session():
+    # clear history
+    llm.clear_history()
+    return '', []
+
+
+def human_response(source_image, history, question_audio, talker_method, voice, rate, volume, pitch,
+                  am, voc, lang, male, 
+                  inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
+                  tts_method, character, 
+                  preprocess_type, is_still_mode, enhancer, batch_size, size_of_image,
+                  pose_style, facerender, exp_weight, blink_every, fps = 20, progress=gr.Progress(track_tqdm=True)):
+    response = history[-1][1]
+    qusetion = history[-1][0]
+    # driven_audio, video_vtt = 'answer.wav', 'answer.vtt'
+    if character == '女性角色':
+        # 女性角色
+        source_image, pic_path = r'./inputs/girl.png', r"./inputs/girl.png"
+        crop_pic_path = "./inputs/first_frame_dir_girl/girl.png"
+        first_coeff_path = "./inputs/first_frame_dir_girl/girl.mat"
+        crop_info = ((403, 403), (19, 30, 502, 513), [40.05956541381802, 40.17324339233366, 443.7892505041507, 443.9029284826663])
+        default_voice = 'zh-CN-XiaoxiaoNeural'
+    elif character == '男性角色':
+        # 男性角色
+        source_image = r'./inputs/boy.png'
+        pic_path = "./inputs/boy.png"
+        crop_pic_path = "./inputs/first_frame_dir_boy/boy.png"
+        first_coeff_path = "./inputs/first_frame_dir_boy/boy.mat"
+        crop_info = ((876, 747), (0, 0, 886, 838), [10.382158280494476, 0, 886, 747.7078990925525])
+        default_voice = 'zh-CN-YunyangNeural'
+    elif character == '自定义角色':
+        if source_image is None:
+            gr.Error("自定义角色需要上传正确的图片")
+            return None
+        default_voice = 'zh-CN-XiaoxiaoNeural'
+    voice = default_voice if not voice else voice
+    # tts.predict(response, voice, rate, volume, pitch, driven_audio, video_vtt)
+    driven_audio, driven_vtt = TTS_response(response, voice, rate, volume, pitch, 
+                 am, voc, lang, male, 
+                 inp_ref, prompt_text, prompt_language, text_language, how_to_cut, question_audio, qusetion, use_mic_voice,
+                 tts_method)
+    if driven_audio is None:
+        gr.Warning("音频没有正常生成，请检查TTS是否正确")
+        return None
+    if talker_method == 'SadTalker':
+        pose_style = random.randint(0, 45)
+        video = talker.test(pic_path,
+                        crop_pic_path,
+                        first_coeff_path,
+                        crop_info,
+                        source_image,
+                        driven_audio,
+                        preprocess_type,
+                        is_still_mode,
+                        enhancer,
+                        batch_size,                            
+                        size_of_image,
+                        pose_style,
+                        facerender,
+                        exp_weight,
+                        use_ref_video,
+                        ref_video,
+                        ref_info,
+                        use_idle_mode,
+                        length_of_audio,
+                        blink_every,
+                        fps=fps)
+    elif talker_method == 'Wav2Lip':
+        video = talker.predict(crop_pic_path, driven_audio, batch_size, enhancer)
+    elif talker_method == 'NeRFTalk':
+        video = talker.predict(driven_audio)
+    else:
+        gr.Warning("不支持的方法：" + talker_method)
+        return None
+    if driven_vtt:
+        return video, driven_vtt
+    else:
+        return video
+
+
+@calculate_time
+def MuseTalker_response(source_video, bbox_shift, question_audio = None, text = '',
+                    voice = 'zh-CN-XiaoxiaoNeural', rate = 0, volume = 100, pitch = 0, 
+                    am = 'fastspeech2', voc = 'pwgan', lang = 'zh', male = False, 
+                    inp_ref = None, prompt_text = "", prompt_language = "", text_language = "", how_to_cut = "", use_mic_voice = False,
+                    tts_method = 'Edge-TTS', batch_size = 4,
+                    progress=gr.Progress(track_tqdm=True)):
+    default_voice = None    
+    voice = default_voice if not voice else voice
+    
+    if not voice:
+        gr.Warning('请选择声音')
+    
+    driven_audio, driven_vtt, _ = LLM_response(question_audio, text, 
+                                               voice, rate, volume, pitch, 
+                                               am, voc, lang, male, 
+                                               inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
+                                               tts_method)
+    print(driven_audio, driven_vtt)
+    video = musetalker.inference_noprepare(driven_audio, 
+                                            source_video, 
+                                            bbox_shift,
+                                            batch_size,
+                                            fps = 25) 
+    
+    if driven_vtt:
+        return (video, driven_vtt)
+    else:
+        return video 
+GPT_SoVITS_ckpt = "GPT_SoVITS/pretrained_models"
+def load_vits_model(gpt_path, sovits_path, progress=gr.Progress(track_tqdm=True)):
+    global vits
+    print("模型加载中...", gpt_path, sovits_path)
+    all_gpt_path, all_sovits_path = os.path.join(GPT_SoVITS_ckpt, gpt_path), os.path.join(GPT_SoVITS_ckpt, sovits_path)
+    vits.load_model(all_gpt_path, all_sovits_path)
+    gr.Info("模型加载成功")
+    return gpt_path, sovits_path
+
+def list_models(dir, endwith = ".pth"):
+    list_folder = os.listdir(dir)
+    list_folder = [i for i in list_folder if i.endswith(endwith)]
+    return list_folder
+
+def character_change(character):
+    if character == '女性角色':
+        # 女性角色
+        source_image = r'./inputs/girl.png'
+    elif character == '男性角色':
+        # 男性角色
+        source_image = r'./inputs/boy.png'
+    elif character == '自定义角色':
+        # gr.Warnings("自定义角色暂未更新，请继续关注后续，可通过自由上传图片模式进行自定义角色")
+        source_image = None
+    return source_image
+
+def webui_setting(talk = False):
+    if not talk:
+        with gr.Tabs():
+            with gr.TabItem('数字人形象设定'):
+                source_image = gr.Image(label="Source image", type="filepath")
+    else:
+        source_image = None
+    with gr.Tabs("TTS Method"):
+        with gr.Accordion("TTS Method语音方法调节 ", open=True):
+            with gr.Tab("Edge-TTS"):
+                voice = gr.Dropdown(edgetts.SUPPORTED_VOICE, 
+                                    value='zh-CN-XiaoxiaoNeural', 
+                                    label="Voice 声音选择")
+                rate = gr.Slider(minimum=-100,
+                                    maximum=100,
+                                    value=0,
+                                    step=1.0,
+                                    label='Rate 速率')
+                volume = gr.Slider(minimum=0,
+                                        maximum=100,
+                                        value=100,
+                                        step=1,
+                                        label='Volume 音量')
+                pitch = gr.Slider(minimum=-100,
+                                    maximum=100,
+                                    value=0,
+                                    step=1,
+                                    label='Pitch 音调')
+            with gr.Tab("PaddleTTS"):
+                am = gr.Dropdown(["FastSpeech2"], label="声学模型选择", value = 'FastSpeech2')
+                voc = gr.Dropdown(["PWGan", "HifiGan"], label="声码器选择", value = 'PWGan')
+                lang = gr.Dropdown(["zh", "en", "mix", "canton"], label="语言选择", value = 'zh')
+                male = gr.Checkbox(label="男声(Male)", value=False)
+            with gr.Tab('GPT-SoVITS'):
+                with gr.Row():
+                    gpt_path = gr.FileExplorer(root = GPT_SoVITS_ckpt, glob = "*.ckpt", value = "s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt", file_count='single', label="GPT模型路径")
+                    sovits_path = gr.FileExplorer(root = GPT_SoVITS_ckpt, glob = "*.pth", value = "s2G488k.pth", file_count='single', label="SoVITS模型路径")
+                    # gpt_path = gr.Dropdown(choices=list_models(GPT_SoVITS_ckpt, 'ckpt'))
+                    # sovits_path = gr.Dropdown(choices=list_models(GPT_SoVITS_ckpt, 'pth'))
+                    # gpt_path = gr.Textbox(label="GPT模型路径", 
+                    #                       value="GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt")
+                    # sovits_path = gr.Textbox(label="SoVITS模型路径", 
+                    #                          value="GPT_SoVITS/pretrained_models/s2G488k.pth")
+                button = gr.Button("加载模型")
+                button.click(fn = load_vits_model, 
+                             inputs=[gpt_path, sovits_path], 
+                             outputs=[gpt_path, sovits_path])
+                
+                with gr.Row():
+                    inp_ref = gr.Audio(label="请上传3~10秒内参考音频，超过会报错！", sources=["microphone", "upload"], type="filepath")
+                    use_mic_voice = gr.Checkbox(label="使用语音问答的麦克风")
+                    prompt_text = gr.Textbox(label="参考音频的文本", value="")
+                    prompt_language = gr.Dropdown(
+                        label="参考音频的语种", choices=["中文", "英文", "日文"], value="中文"
+                    )
+                asr_button = gr.Button("语音识别 - 克隆参考音频")
+                asr_button.click(fn=Asr,inputs=[inp_ref],outputs=[prompt_text])
+                with gr.Row():
+                    text_language = gr.Dropdown(
+                        label="需要合成的语种", choices=["中文", "英文", "日文", "中英混合", "日英混合", "多语种混合"], value="中文"
+                    )
+                    
+                    how_to_cut = gr.Dropdown(
+                        label="怎么切",
+                        choices=["不切", "凑四句一切", "凑50字一切", "按中文句号。切", "按英文句号.切", "按标点符号切" ],
+                        value="凑四句一切",
+                        interactive=True,
+                    )
+            
+            with gr.Column(variant='panel'): 
+                batch_size = gr.Slider(minimum=1,
+                                    maximum=10,
+                                    value=2,
+                                    step=1,
+                                    label='Talker Batch size')
+
+    character = gr.Radio(['女性角色', 
+                          '男性角色', 
+                          '自定义角色'], 
+                         label="角色选择", value='自定义角色')
+    character.change(fn = character_change, inputs=[character], outputs = [source_image])
+    tts_method = gr.Radio(['Edge-TTS', 'PaddleTTS', 'GPT-SoVITS克隆声音', 'Comming Soon!!!'], label="Text To Speech Method", 
+                                              value = 'Edge-TTS')
+    tts_method.change(fn = tts_model_change, inputs=[tts_method], outputs = [tts_method])
+    asr_method = gr.Radio(choices = ['Whisper-tiny', 'Whisper-base', 'FunASR', 'Comming Soon!!!'], value='Whisper-base', label = '语音识别模型选择')
+    asr_method.change(fn = asr_model_change, inputs=[asr_method], outputs = [asr_method])
+    talker_method = gr.Radio(choices = ['SadTalker', 'Wav2Lip', 'NeRFTalk', 'Comming Soon!!!'], 
+                      value = 'SadTalker', label = '数字人模型选择')
+    talker_method.change(fn = talker_model_change, inputs=[talker_method], outputs = [talker_method])
+    llm_method = gr.Dropdown(choices = ['Qwen', 'Qwen2', 'Linly', 'Gemini', 'ChatGLM', 'ChatGPT', 'GPT4Free', '直接回复 Direct Reply', 'Comming Soon!!!'], value = '直接回复 Direct Reply', label = 'LLM 模型选择')
+    llm_method.change(fn = llm_model_change, inputs=[llm_method], outputs = [llm_method])
+    return  (source_image, voice, rate, volume, pitch, 
+             am, voc, lang, male, 
+             inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
+             tts_method, batch_size, character, talker_method, asr_method, llm_method)
+
+
+def exmaple_setting(asr, text, character, talk , tts, voice, llm):
+    # 默认text的Example
+    examples =  [
+        ['Whisper-base', '应对压力最有效的方法是什么？', '女性角色', 'SadTalker', 'Edge-TTS', 'zh-CN-XiaoxiaoNeural', '直接回复 Direct Reply'],
+        ['Whisper-tiny', '应对压力最有效的方法是什么？', '女性角色', 'SadTalker', 'PaddleTTS', 'None', '直接回复 Direct Reply'],
+        ['Whisper-base', '应对压力最有效的方法是什么？', '女性角色', 'SadTalker', 'Edge-TTS', 'zh-CN-XiaoxiaoNeural', 'Qwen'],
+        ['FunASR', '如何进行时间管理？','男性角色', 'SadTalker', 'Edge-TTS', 'zh-CN-YunyangNeural', 'Qwen'],
+        ['Whisper-tiny', '为什么有些人选择使用纸质地图或寻求方向，而不是依赖GPS设备或智能手机应用程序？','女性角色', 'Wav2Lip', 'PaddleTTS', 'None', 'Qwen'],
+        ]
+
+    with gr.Row(variant='panel'):
+        with gr.Column(variant='panel'):
+            gr.Markdown("## Test Examples")
+            gr.Examples(
+                examples = examples,
+                inputs = [asr, text, character, talk , tts, voice, llm],
+            )
+def app():
+    with gr.Blocks(analytics_enabled=False, title = 'Linly-Talker') as inference:
+        gr.HTML(get_title("Linly 智能对话系统 (Linly-Talker) 文本/语音对话"))
+        with gr.Row(equal_height=False):
+            with gr.Column(variant='panel'): 
+                (source_image, voice, rate, volume, pitch, 
+                am, voc, lang, male, 
+                inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
+                tts_method, batch_size, character, talker_method, asr_method, llm_method)= webui_setting()
+             
+            
+            with gr.Column(variant='panel'):
+                with gr.Tabs():
+                    with gr.TabItem('对话'):
+                        with gr.Group():
+                            question_audio = gr.Audio(sources=['microphone','upload'], type="filepath", label = '语音对话')
+                            input_text = gr.Textbox(label="输入文字/问题", lines=3)
+                            asr_text = gr.Button('语音识别（语音对话后点击）')
+                        asr_text.click(fn=Asr,inputs=[question_audio],outputs=[input_text])
+                # with gr.TabItem('SadTalker数字人参数设置'):
+                #     with gr.Accordion("Advanced Settings",
+                #                     open=False):
+                #         gr.Markdown("SadTalker: need help? please visit our [[best practice page](https://github.com/OpenTalker/SadTalker/blob/main/docs/best_practice.md)] for more detials")
+                #         with gr.Column(variant='panel'):
+                #             # width = gr.Slider(minimum=64, elem_id="img2img_width", maximum=2048, step=8, label="Manually Crop Width", value=512) # img2img_width
+                #             # height = gr.Slider(minimum=64, elem_id="img2img_height", maximum=2048, step=8, label="Manually Crop Height", value=512) # img2img_width
+                #             with gr.Row():
+                #                 pose_style = gr.Slider(minimum=0, maximum=45, step=1, label="Pose style", value=0) #
+                #                 exp_weight = gr.Slider(minimum=0, maximum=3, step=0.1, label="expression scale", value=1) # 
+                #                 blink_every = gr.Checkbox(label="use eye blink", value=True)
+
+                #             with gr.Row():
+                #                 size_of_image = gr.Radio([256, 512], value=256, label='face model resolution', info="use 256/512 model? 256 is faster") # 
+                #                 preprocess_type = gr.Radio(['crop', 'resize','full'], value='full', label='preprocess', info="How to handle input image?")
+                            
+                #             with gr.Row():
+                #                 is_still_mode = gr.Checkbox(label="Still Mode (fewer head motion, works with preprocess `full`)")
+                #                 facerender = gr.Radio(['facevid2vid'], value='facevid2vid', label='facerender', info="which face render?")
+                                
+                #             with gr.Row():
+                #                 # batch_size = gr.Slider(label="batch size in generation", step=1, maximum=10, value=1)
+                #                 fps = gr.Slider(label='fps in generation', step=1, maximum=30, value =20)
+                #                 enhancer = gr.Checkbox(label="GFPGAN as Face enhancer(slow)")       
+                with gr.Tabs():
+                    with gr.TabItem('数字人问答'):
+                        gen_video = gr.Video(label="生成视频", format="mp4", autoplay=False)
+                video_button = gr.Button("🎬 生成数字人视频", variant='primary')
+            video_button.click(fn=Talker_response,inputs=[question_audio, talker_method, input_text, voice, rate, volume, pitch,
+                                                          am, voc, lang, male, 
+                                                          inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
+                                                          tts_method, batch_size, character],outputs=[gen_video])
+        exmaple_setting(asr_method, input_text, character, talker_method, tts_method, voice, llm_method)
+    return inference
+
+def app_multi():
+    with gr.Blocks(analytics_enabled=False, title = 'Linly-Talker') as inference:
+        gr.HTML(get_title("Linly 智能对话系统 (Linly-Talker) 多轮GPT对话"))
+        with gr.Row():
+            with gr.Column():
+                (source_image, voice, rate, volume, pitch, 
+                am, voc, lang, male, 
+                inp_ref, prompt_text, prompt_language, text_language, how_to_cut,  use_mic_voice,
+                tts_method, batch_size, character, talker_method, asr_method, llm_method)= webui_setting()
+                video = gr.Video(label = '数字人问答', scale = 0.5)
+                video_button = gr.Button("🎬 生成数字人视频（对话后）", variant = 'primary')
+            
+            with gr.Column():
+                with gr.Tabs(elem_id="sadtalker_checkbox"):
+                    with gr.TabItem('SadTalker数字人参数设置'):
+                        with gr.Accordion("Advanced Settings",
+                                        open=False):
+                            gr.Markdown("SadTalker: need help? please visit our [[best practice page](https://github.com/OpenTalker/SadTalker/blob/main/docs/best_practice.md)] for more detials")
+                            with gr.Column(variant='panel'):
+                                # width = gr.Slider(minimum=64, elem_id="img2img_width", maximum=2048, step=8, label="Manually Crop Width", value=512) # img2img_width
+                                # height = gr.Slider(minimum=64, elem_id="img2img_height", maximum=2048, step=8, label="Manually Crop Height", value=512) # img2img_width
+                                with gr.Row():
+                                    pose_style = gr.Slider(minimum=0, maximum=45, step=1, label="Pose style", value=0) #
+                                    exp_weight = gr.Slider(minimum=0, maximum=3, step=0.1, label="expression scale", value=1) # 
+                                    blink_every = gr.Checkbox(label="use eye blink", value=True)
+
+                                with gr.Row():
+                                    size_of_image = gr.Radio([256, 512], value=256, label='face model resolution', info="use 256/512 model? 256 is faster") # 
+                                    preprocess_type = gr.Radio(['crop', 'resize','full', 'extcrop', 'extfull'], value='crop', label='preprocess', info="How to handle input image?")
+                                
+                                with gr.Row():
+                                    is_still_mode = gr.Checkbox(label="Still Mode (fewer head motion, works with preprocess `full`)")
+                                    facerender = gr.Radio(['facevid2vid'], value='facevid2vid', label='facerender', info="which face render?")
+                                    
+                                with gr.Row():
+                                    fps = gr.Slider(label='fps in generation', step=1, maximum=30, value =20)
+                                    enhancer = gr.Checkbox(label="GFPGAN as Face enhancer(slow)")   
+                with gr.Row():
+                    with gr.Column(scale=3):
+                        system_input = gr.Textbox(value=default_system, lines=1, label='System (设定角色)')
+                    with gr.Column(scale=1):
+                        modify_system = gr.Button("🛠️ 设置system并清除历史对话", scale=2)
+                    system_state = gr.Textbox(value=default_system, visible=False)
+
+                chatbot = gr.Chatbot(height=400, show_copy_button=True)
+                with gr.Group():
+                    question_audio = gr.Audio(sources=['microphone','upload'], type="filepath", label='语音对话', autoplay=False)
+                    asr_text = gr.Button('🎤 语音识别（语音对话后点击）')
+                
+                # 创建一个文本框组件，用于输入 prompt。
+                msg = gr.Textbox(label="Prompt/问题")
+                asr_text.click(fn=Asr,inputs=[question_audio],outputs=[msg])
+                
+                with gr.Row():
+                    clear_history = gr.Button("🧹 清除历史对话")
+                    sumbit = gr.Button("🚀 发送", variant = 'primary')
+                    
+            # 设置按钮的点击事件。当点击时，调用上面定义的 函数，并传入用户的消息和聊天历史记录，然后更新文本框和聊天机器人组件。
+            sumbit.click(chat_response, inputs=[system_input, msg, chatbot], 
+                         outputs=[msg, chatbot])
+            
+            # 点击后清空后端存储的聊天记录
+            clear_history.click(fn = clear_session, outputs = [msg, chatbot])
+            
+            # 设置system并清除历史对话
+            modify_system.click(fn=modify_system_session,
+                        inputs=[system_input],
+                        outputs=[system_state, system_input, chatbot])
+            
+            video_button.click(fn = human_response, inputs = [source_image, chatbot, question_audio, talker_method, voice, rate, volume, pitch,
+                                                             am, voc, lang, male, 
+                                                             inp_ref, prompt_text, prompt_language, text_language, how_to_cut,  use_mic_voice, 
+                                                             tts_method, character,preprocess_type, 
+                                                             is_still_mode, enhancer, batch_size, size_of_image,
+                                                             pose_style, facerender, exp_weight, blink_every, fps], outputs = [video])
+
+        exmaple_setting(asr_method, msg, character, talker_method, tts_method, voice, llm_method)
+    return inference
+
+def app_img():
+    with gr.Blocks(analytics_enabled=False, title = 'Linly-Talker') as inference:
+        gr.HTML(get_title("Linly 智能对话系统 (Linly-Talker) 个性化角色互动"))
+        with gr.Row(equal_height=False):
+            with gr.Column(variant='panel'):                                
+                (source_image, voice, rate, volume, pitch, 
+                am, voc, lang, male, 
+                inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
+                tts_method, batch_size, character, talker_method, asr_method, llm_method)= webui_setting()
+                                
+            # driven_audio = 'answer.wav'           
+            with gr.Column(variant='panel'): 
+                with gr.Tabs():
+                    with gr.TabItem('对话'):
+                        with gr.Group():
+                            question_audio = gr.Audio(sources=['microphone','upload'], type="filepath", label = '语音对话')
+                            input_text = gr.Textbox(label="输入文字/问题", lines=3)
+                            asr_text = gr.Button('语音识别（语音对话后点击）')
+                        asr_text.click(fn=Asr,inputs=[question_audio],outputs=[input_text])
+                with gr.Tabs(elem_id="text_examples"): 
+                    gr.Markdown("## Text Examples")
+                    examples =  [
+                        ['应对压力最有效���方法是什么？'],
+                        ['如何进行时间管理？'],
+                        ['为什么有些人选择使用纸质地图或寻求方向，而不是依赖GPS设备或智能手机应用程序？'],
+                    ]
+                    gr.Examples(
+                        examples = examples,
+                        inputs = [input_text],
+                    )
+                with gr.Tabs(elem_id="sadtalker_checkbox"):
+                    with gr.TabItem('SadTalker数字人参数设置'):
+                        with gr.Accordion("Advanced Settings",
+                                        open=False):
+                            gr.Markdown("SadTalker: need help? please visit our [[best practice page](https://github.com/OpenTalker/SadTalker/blob/main/docs/best_practice.md)] for more detials")
+                            with gr.Column(variant='panel'):
+                                # width = gr.Slider(minimum=64, elem_id="img2img_width", maximum=2048, step=8, label="Manually Crop Width", value=512) # img2img_width
+                                # height = gr.Slider(minimum=64, elem_id="img2img_height", maximum=2048, step=8, label="Manually Crop Height", value=512) # img2img_width
+                                with gr.Row():
+                                    pose_style = gr.Slider(minimum=0, maximum=45, step=1, label="Pose style", value=0) #
+                                    exp_weight = gr.Slider(minimum=0, maximum=3, step=0.1, label="expression scale", value=1) # 
+                                    blink_every = gr.Checkbox(label="use eye blink", value=True)
+
+                                with gr.Row():
+                                    size_of_image = gr.Radio([256, 512], value=256, label='face model resolution', info="use 256/512 model? 256 is faster") # 
+                                    preprocess_type = gr.Radio(['crop', 'resize','full', 'extcrop', 'extfull'], value='crop', label='preprocess', info="How to handle input image?")
+                                
+                                with gr.Row():
+                                    is_still_mode = gr.Checkbox(label="Still Mode (fewer head motion, works with preprocess `full`)")
+                                    facerender = gr.Radio(['facevid2vid'], value='facevid2vid', label='facerender', info="which face render?")
+                                    
+                                with gr.Row():
+                                    fps = gr.Slider(label='fps in generation', step=1, maximum=30, value =20)
+                                    enhancer = gr.Checkbox(label="GFPGAN as Face enhancer(slow)")                                               
+
+                with gr.Tabs(elem_id="sadtalker_genearted"):
+                    gen_video = gr.Video(label="数字人视频", format="mp4")
+
+                submit = gr.Button('🎬 生成数字人视频', elem_id="sadtalker_generate", variant='primary')
+            submit.click(
+                fn=Talker_response_img,
+                inputs=[question_audio,
+                        talker_method, 
+                        input_text,
+                        voice, rate, volume, pitch,
+                        am, voc, lang, male, 
+                        inp_ref, prompt_text, prompt_language, text_language, how_to_cut,  use_mic_voice,
+                        tts_method,
+                        source_image, 
+                        preprocess_type,
+                        is_still_mode,
+                        enhancer,
+                        batch_size,                            
+                        size_of_image,
+                        pose_style,
+                        facerender,
+                        exp_weight,
+                        blink_every,
+                        fps], 
+                outputs=[gen_video]
+                )
+        
+        with gr.Row():
+            examples = [
+                [
+                    'examples/source_image/full_body_2.png', 'SadTalker',
+                    'crop',
+                    False,
+                    False
+                ],
+                [
+                    'examples/source_image/full_body_1.png', 'SadTalker',
+                    'full',
+                    True,
+                    False
+                ],
+                [
+                    'examples/source_image/full4.jpeg', 'SadTalker',
+                    'crop',
+                    False,
+                    True
+                ],
+            ]
+            gr.Examples(examples=examples,
+                        inputs=[
+                            source_image, talker_method,
+                            preprocess_type,
+                            is_still_mode,
+                            enhancer], 
+                        outputs=[gen_video],
+                        # cache_examples=True,
+                        )
+    return inference
+
+def app_vits():
+    with gr.Blocks(analytics_enabled=False, title = 'Linly-Talker') as inference:
+        gr.HTML(get_title("Linly 智能对话系统 (Linly-Talker) 语音克隆"))
+        with gr.Row(equal_height=False):
+            with gr.Column(variant='panel'): 
+                (source_image, voice, rate, volume, pitch, 
+                am, voc, lang, male, 
+                inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
+                tts_method, batch_size, character, talker_method, asr_method, llm_method)= webui_setting()
+            with gr.Column(variant='panel'): 
+                with gr.Tabs():
+                    with gr.TabItem('对话'):
+                        with gr.Group():
+                            question_audio = gr.Audio(sources=['microphone','upload'], type="filepath", label = '语音对话')
+                            input_text = gr.Textbox(label="输入文字/问题", lines=3)
+                            asr_text = gr.Button('语音识别（语音对话后点击）')
+                        asr_text.click(fn=Asr,inputs=[question_audio],outputs=[input_text])
+                with gr.Tabs():
+                    with gr.TabItem('数字人问答'):
+                        gen_video = gr.Video(label="数字人视频", format="mp4", autoplay=False)
+                video_button = gr.Button("🎬 生成数字人视频", variant='primary')
+            video_button.click(fn=Talker_response,inputs=[question_audio, talker_method, input_text, voice, rate, volume, pitch, am, voc, lang, male, 
+                            inp_ref, prompt_text, prompt_language, text_language, how_to_cut,  use_mic_voice,
+                            tts_method, batch_size, character],outputs=[gen_video])
+        exmaple_setting(asr_method, input_text, character, talker_method, tts_method, voice, llm_method)
+    return inference
+
+def app_talk():
+    with gr.Blocks(analytics_enabled=False, title = 'Linly-Talker') as inference:
+        gr.HTML(get_title("Linly 智能对话系统 (Linly-Talker) 数字人播报"))
+        with gr.Row(equal_height=False):
+            with gr.Column(variant='panel'): 
+                with gr.Tabs():
+                    with gr.Tab("图片人物"):
+                        source_image = gr.Image(label='Source image', type = 'filepath')
+                        
+                    with gr.Tab("视频人物"):
+                        source_video = gr.Video(label="Source video")
+               
+                (_, voice, rate, volume, pitch, 
+                am, voc, lang, male, 
+                inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
+                tts_method, batch_size, character, talker_method, asr_method, llm_method)= webui_setting()
+        
+            with gr.Column(variant='panel'):
+                with gr.Tabs():
+                    with gr.TabItem('对话'):
+                        with gr.Group():
+                            question_audio = gr.Audio(sources=['microphone','upload'], type="filepath", label = '语音对话')
+                            input_text = gr.Textbox(label="输入文字/问题", lines=3)
+                            asr_text = gr.Button('语音识别（语音对话后点击）')
+                        asr_text.click(fn=Asr,inputs=[question_audio],outputs=[input_text]) 
+                with gr.Tabs():
+                    with gr.TabItem('SadTalker数字人参数设置'):
+                        with gr.Accordion("Advanced Settings",
+                                        open=False):
+                            gr.Markdown("SadTalker: need help? please visit our [[best practice page](https://github.com/OpenTalker/SadTalker/blob/main/docs/best_practice.md)] for more detials")
+                            with gr.Column(variant='panel'):
+                                # width = gr.Slider(minimum=64, elem_id="img2img_width", maximum=2048, step=8, label="Manually Crop Width", value=512) # img2img_width
+                                # height = gr.Slider(minimum=64, elem_id="img2img_height", maximum=2048, step=8, label="Manually Crop Height", value=512) # img2img_width
+                                with gr.Row():
+                                    pose_style = gr.Slider(minimum=0, maximum=45, step=1, label="Pose style", value=0) #
+                                    exp_weight = gr.Slider(minimum=0, maximum=3, step=0.1, label="expression scale", value=1) # 
+                                    blink_every = gr.Checkbox(label="use eye blink", value=True)
+
+                                with gr.Row():
+                                    size_of_image = gr.Radio([256, 512], value=256, label='face model resolution', info="use 256/512 model? 256 is faster") # 
+                                    preprocess_type = gr.Radio(['crop', 'resize','full'], value='full', label='preprocess', info="How to handle input image?")
+                                
+                                with gr.Row():
+                                    is_still_mode = gr.Checkbox(label="Still Mode (fewer head motion, works with preprocess `full`)")
+                                    facerender = gr.Radio(['facevid2vid'], value='facevid2vid', label='facerender', info="which face render?")
+                                    
+                                with gr.Row():
+                                    # batch_size = gr.Slider(label="batch size in generation", step=1, maximum=10, value=1)
+                                    fps = gr.Slider(label='fps in generation', step=1, maximum=30, value =20)
+                                    enhancer = gr.Checkbox(label="GFPGAN as Face enhancer(slow)")                                               
+
+                with gr.Tabs():
+                    gen_video = gr.Video(label="数字人视频", format="mp4")
+
+                video_button = gr.Button('🎬 生成数字人视频', elem_id="sadtalker_generate", variant='primary')
+
+                video_button.click(fn=Talker_Say,inputs=[preprocess_type, is_still_mode, enhancer, batch_size, size_of_image,
+                                pose_style, facerender, exp_weight, blink_every, fps,
+                                source_image, source_video, question_audio, talker_method, input_text, voice, rate, volume, pitch, am, voc, lang, male, 
+                                inp_ref, prompt_text, prompt_language, text_language, how_to_cut,  use_mic_voice,
+                                tts_method, character],outputs=[gen_video])
+            
+        with gr.Row():
+            with gr.Column(variant='panel'):
+                gr.Markdown("## Test Examples")
+                gr.Examples(
+                    examples = [
+                        [
+                            'examples/source_image/full_body_2.png',
+                            '应对压力最有效的方法是什么？',
+                        ],
+                        [
+                            'examples/source_image/full_body_1.png',
+                            '如何进行时间管理？',
+                        ],
+                        [
+                            'examples/source_image/full3.png',
+                            '为什么有些人选择使用纸质地图或寻求方向，而不是依赖GPS设备或智能手机应用程序？',
+                        ],
+                    ],
+                    fn = Talker_Say,
+                    inputs = [source_image, input_text],
+                )   
+    return inference
+
+def load_musetalk_model():
+    gr.Warning("若显存不足，可能会导致模型加载失败，可以尝试使用其他摸型或者换其他设备尝试。")
+    gr.Info("MuseTalk模型导入中...")
+    musetalker.init_model()
+    gr.Info("MuseTalk模型导入成功")
+    return "MuseTalk模型导入成功"
+def musetalk_prepare_material(source_video, bbox_shift):
+    if musetalker.load is False:
+        gr.Warning("请先加载MuseTalk模型后重新上传文件")
+        return source_video, None
+    return musetalker.prepare_material(source_video, bbox_shift)
+def app_muse():
+    with gr.Blocks(analytics_enabled=False, title = 'Linly-Talker') as inference:
+        gr.HTML(get_title("Linly 智能对话系统 (Linly-Talker) MuseTalker数字人实时对话"))
+        with gr.Row(equal_height=False):
+            with gr.Column(variant='panel'): 
+                with gr.TabItem('MuseV Video'):
+                    gr.Markdown("MuseV: need help? please visit MuseVDemo to generate Video https://huggingface.co/spaces/AnchorFake/MuseVDemo")
+                    with gr.Row():
+                        source_video = gr.Video(label="Reference Video",sources=['upload'])
+                    gr.Markdown("BBox_shift 推荐值下限，在生成初始结果后生成相应的 bbox 范围。如果结果不理想，可以根据该参考值进行调整。\n一般来说，在我们的实验观察中，我们发现正值（向下半部分移动）通常会增加嘴巴的张开度，而负值（向上半部分移动）通常会减少嘴巴的张开度。然而，需要注意的是，这并不是绝对的规则，用户可能需要根据他们的具体需求和期望效果来调整该参数。")
+                    with gr.Row():
+                        bbox_shift = gr.Number(label="BBox_shift value, px", value=0)
+                        bbox_shift_scale = gr.Textbox(label="bbox_shift_scale", 
+                                                        value="",interactive=False)
+                load_musetalk = gr.Button("加载MuseTalk模型(传入视频前先加载)", variant='primary')
+                load_musetalk.click(fn=load_musetalk_model, outputs=bbox_shift_scale)
+
+                # (_, voice, rate, volume, pitch, 
+                # am, voc, lang, male, 
+                # inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
+                # tts_method, batch_size, character, talker_method, asr_method, llm_method)= webui_setting()
+                with gr.Tabs("TTS Method"):
+                    with gr.Accordion("TTS Method语音方法调节 ", open=True):
+                        with gr.Tab("Edge-TTS"):
+                            voice = gr.Dropdown(edgetts.SUPPORTED_VOICE, 
+                                                value='zh-CN-XiaoxiaoNeural', 
+                                                label="Voice 声音选择")
+                            rate = gr.Slider(minimum=-100,
+                                                maximum=100,
+                                                value=0,
+                                                step=1.0,
+                                                label='Rate 速率')
+                            volume = gr.Slider(minimum=0,
+                                                    maximum=100,
+                                                    value=100,
+                                                    step=1,
+                                                    label='Volume 音量')
+                            pitch = gr.Slider(minimum=-100,
+                                                maximum=100,
+                                                value=0,
+                                                step=1,
+                                                label='Pitch 音调')
+                        with gr.Tab("PaddleTTS"):
+                            am = gr.Dropdown(["FastSpeech2"], label="声学模型选择", value = 'FastSpeech2')
+                            voc = gr.Dropdown(["PWGan", "HifiGan"], label="声码器选择", value = 'PWGan')
+                            lang = gr.Dropdown(["zh", "en", "mix", "canton"], label="语言选择", value = 'zh')
+                            male = gr.Checkbox(label="男声(Male)", value=False)
+                        with gr.Tab('GPT-SoVITS'):
+                            with gr.Row():
+                                gpt_path = gr.FileExplorer(root = GPT_SoVITS_ckpt, glob = "*.ckpt", value = "s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt", file_count='single', label="GPT模型路径")
+                                sovits_path = gr.FileExplorer(root = GPT_SoVITS_ckpt, glob = "*.pth", value = "s2G488k.pth", file_count='single', label="SoVITS模型路径")
+                                # gpt_path = gr.Dropdown(choices=list_models(GPT_SoVITS_ckpt, 'ckpt'))
+                                # sovits_path = gr.Dropdown(choices=list_models(GPT_SoVITS_ckpt, 'pth'))
+                                # gpt_path = gr.Textbox(label="GPT模型路径", 
+                                #                       value="GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt")
+                                # sovits_path = gr.Textbox(label="SoVITS模型路径", 
+                                #                          value="GPT_SoVITS/pretrained_models/s2G488k.pth")
+                            button = gr.Button("加载模型")
+                            button.click(fn = load_vits_model, 
+                                        inputs=[gpt_path, sovits_path], 
+                                        outputs=[gpt_path, sovits_path])
+                            
+                            with gr.Row():
+                                inp_ref = gr.Audio(label="请上传3~10秒内参考音频，超过会报错！", sources=["microphone", "upload"], type="filepath")
+                                use_mic_voice = gr.Checkbox(label="使用语音问答的麦克风")
+                                prompt_text = gr.Textbox(label="参考音频的文本", value="")
+                                prompt_language = gr.Dropdown(
+                                    label="参考音频的语种", choices=["中文", "英文", "日文"], value="中文"
+                                )
+                            asr_button = gr.Button("语音识别 - 克隆参考音频")
+                            asr_button.click(fn=Asr,inputs=[inp_ref],outputs=[prompt_text])
+                            with gr.Row():
+                                text_language = gr.Dropdown(
+                                    label="需要合成的语种", choices=["中文", "英文", "日文", "中英混合", "日英混合", "多语种混合"], value="中文"
+                                )
+                                
+                                how_to_cut = gr.Dropdown(
+                                    label="怎么切",
+                                    choices=["不切", "凑四句一切", "凑50字一切", "按中文句号。切", "按英文句号.切", "按标点符号切" ],
+                                    value="凑四句一切",
+                                    interactive=True,
+                                )
+                        
+                        with gr.Column(variant='panel'): 
+                            batch_size = gr.Slider(minimum=1,
+                                                maximum=10,
+                                                value=2,
+                                                step=1,
+                                                label='Talker Batch size')
+
+                tts_method = gr.Radio(['Edge-TTS', 'PaddleTTS', 'GPT-SoVITS克隆声音', 'Comming Soon!!!'], label="Text To Speech Method", 
+                                                        value = 'Edge-TTS')
+                tts_method.change(fn = tts_model_change, inputs=[tts_method], outputs = [tts_method])
+                asr_method = gr.Radio(choices = ['Whisper-tiny', 'Whisper-base', 'FunASR', 'Comming Soon!!!'], value='Whisper-base', label = '语音识别模型选择')
+                asr_method.change(fn = asr_model_change, inputs=[asr_method], outputs = [asr_method])
+                llm_method = gr.Dropdown(choices = ['Qwen', 'Qwen2', 'Linly', 'Gemini', 'ChatGLM', 'ChatGPT', 'GPT4Free', '直接回复 Direct Reply', 'Comming Soon!!!'], value = '直接回复 Direct Reply', label = 'LLM 模型选择')
+                llm_method.change(fn = llm_model_change, inputs=[llm_method], outputs = [llm_method])
+            
+            source_video.change(fn=musetalk_prepare_material, inputs=[source_video, bbox_shift], outputs=[source_video, bbox_shift_scale])
+            
+            with gr.Column(variant='panel'):
+                with gr.Tabs():
+                    with gr.TabItem('对话'):
+                        with gr.Group():
+                            question_audio = gr.Audio(sources=['microphone','upload'], type="filepath", label = '语音对话')
+                            input_text = gr.Textbox(label="输入文字/问题", lines=3)
+                            asr_text = gr.Button('语音识别（语音对话后点击）')
+                        asr_text.click(fn=Asr,inputs=[question_audio],outputs=[input_text]) 
+            
+                with gr.TabItem("MuseTalk Video"):
+                    gen_video = gr.Video(label="数字人视频", format="mp4")
+                submit = gr.Button('Generate', elem_id="sadtalker_generate", variant='primary')
+                examples = [os.path.join('Musetalk/data/video', video) for video in os.listdir("Musetalk/data/video")]
+                # ['Musetalk/data/video/yongen_musev.mp4', 'Musetalk/data/video/musk_musev.mp4', 'Musetalk/data/video/monalisa_musev.mp4', 'Musetalk/data/video/sun_musev.mp4', 'Musetalk/data/video/seaside4_musev.mp4', 'Musetalk/data/video/sit_musev.mp4', 'Musetalk/data/video/man_musev.mp4']
+                
+                gr.Markdown("## MuseV Video Examples")
+                gr.Examples(
+                    examples=[
+                        ['Musetalk/data/video/yongen_musev.mp4', 5],
+                        ['Musetalk/data/video/musk_musev.mp4', 5],
+                        ['Musetalk/data/video/monalisa_musev.mp4', 5],
+                        ['Musetalk/data/video/sun_musev.mp4', 5],
+                        ['Musetalk/data/video/seaside4_musev.mp4', 5],
+                        ['Musetalk/data/video/sit_musev.mp4', 5],
+                        ['Musetalk/data/video/man_musev.mp4', 5]
+                        ],
+                    inputs =[source_video, bbox_shift], 
+                )
+
+            submit.click(
+                fn=MuseTalker_response,
+                inputs=[source_video, bbox_shift, question_audio, input_text, voice, rate, volume, pitch, am, voc, lang, male, 
+                            inp_ref, prompt_text, prompt_language, text_language, how_to_cut,  use_mic_voice,
+                            tts_method, batch_size], 
+                outputs=[gen_video]
+                )
+    return inference
+def asr_model_change(model_name, progress=gr.Progress(track_tqdm=True)):
+    global asr
+
+    # 清理显存，在加载新的模型之前释放不必要的显存
+    clear_memory()
+
+    if model_name == "Whisper-tiny":
+        try:
+            if os.path.exists('Whisper/tiny.pt'):
+                asr = WhisperASR('Whisper/tiny.pt')
+            else:
+                asr = WhisperASR('tiny')
+            gr.Info("Whisper-tiny模型导入成功")
+        except Exception as e:
+            gr.Warning(f"Whisper-tiny模型下载失败 {e}")
+    elif model_name == "Whisper-base":
+        try:
+            if os.path.exists('Whisper/base.pt'):
+                asr = WhisperASR('Whisper/base.pt')
+            else:
+                asr = WhisperASR('base')
+            gr.Info("Whisper-base模型导入成功")
+        except Exception as e:
+            gr.Warning(f"Whisper-base模型下载失败 {e}")
+    elif model_name == 'FunASR':
+        try:
+            from ASR import FunASR
+            asr = FunASR()
+            gr.Info("FunASR模型导入成功")
+        except Exception as e:
+            gr.Warning(f"FunASR模型下载失败 {e}")
+    else:
+        gr.Warning("未知ASR模型，可提issue和PR 或者 建议更新模型")
+    return model_name
+
+def llm_model_change(model_name, progress=gr.Progress(track_tqdm=True)):
+    global llm
+    gemini_apikey = ""
+    openai_apikey = ""
+    proxy_url = None
+
+    # 清理显存，在加载新的模型之前释放不必要的显存
+    clear_memory()
+
+    if model_name == 'Linly':
+        try:
+            llm = llm_class.init_model('Linly', 'Linly-AI/Chinese-LLaMA-2-7B-hf', prefix_prompt=prefix_prompt)
+            gr.Info("Linly模型导入成功")
+        except Exception as e:
+            gr.Warning(f"Linly模型下载失败 {e}")
+    elif model_name == 'Qwen':
+        try:
+            llm = llm_class.init_model('Qwen', 'Qwen/Qwen-1_8B-Chat', prefix_prompt=prefix_prompt)
+            gr.Info("Qwen模型导入成功")
+        except Exception as e:
+            gr.Warning(f"Qwen模型下载失败 {e}")
+    elif model_name == 'Qwen2':
+        try:
+            llm = llm_class.init_model('Qwen2', 'Qwen/Qwen1.5-0.5B-Chat', prefix_prompt=prefix_prompt)
+            gr.Info("Qwen2模型导入成功")
+        except Exception as e:
+            gr.Warning(f"Qwen2模型下载失败 {e}")
+    elif model_name == 'Gemini':
+        if gemini_apikey:
+            llm = llm_class.init_model('Gemini', 'gemini-pro', gemini_apikey, proxy_url)
+            gr.Info("Gemini模型导入成功")
+        else:
+            gr.Warning("请填写Gemini的api_key")
+    elif model_name == 'ChatGLM':
+        try:
+            llm = llm_class.init_model('ChatGLM', 'THUDM/chatglm3-6b', prefix_prompt=prefix_prompt)
+            gr.Info("ChatGLM模型导入成功")
+        except Exception as e:
+            gr.Warning(f"ChatGLM模型导入失败 {e}")
+    elif model_name == 'ChatGPT':
+        if openai_apikey:
+            llm = llm_class.init_model('ChatGPT', api_key=openai_apikey, proxy_url=proxy_url, prefix_prompt=prefix_prompt)
+        else:
+            gr.Warning("请填写OpenAI的api_key")
+    elif model_name == '直接回复 Direct Reply':
+        llm =llm_class.init_model(model_name)
+        gr.Info("直接回复，不实用LLM模型")
+    elif model_name == 'GPT4Free':
+        try:
+            llm = llm_class.init_model('GPT4Free', prefix_prompt=prefix_prompt)
+            gr.Info("GPT4Free模型导入成功, 请注意GPT4Free可能不稳定")
+        except Exception as e:
+            gr.Warning(f"GPT4Free模型下载失败 {e}")
+    else:
+        gr.Warning("未知LLM模型，可提issue和PR 或者 建议更新模型")
+    return model_name
+    
+def talker_model_change(model_name, progress=gr.Progress(track_tqdm=True)):
+    global talker
+
+    # 清理显存，在加载新的模型之前释放不必要的显存
+    clear_memory()
+
+    if model_name not in ['SadTalker', 'Wav2Lip', 'NeRFTalk']:
+        gr.Warning("其他模型还未集成，请等待")
+    if model_name == 'SadTalker':
+        try:
+            from TFG import SadTalker
+            talker = SadTalker(lazy_load=True)
+            gr.Info("SadTalker模型导入成功")
+        except Exception as e:
+            gr.Warning("SadTalker模型加载失败", e)
+    elif model_name == 'Wav2Lip':
+        try:
+            from TFG import Wav2Lip
+            clear_memory()
+            talker = Wav2Lip("checkpoints/wav2lip_gan.pth")
+            gr.Info("Wav2Lip模型导入成功")
+        except Exception as e:
+            gr.Warning("Wav2Lip模型加载失败", e)
+    elif model_name == 'NeRFTalk':
+        try:
+            from TFG import ERNeRF
+            talker = ERNeRF()
+            talker.init_model('checkpoints/Obama_ave.pth', 'checkpoints/Obama.json')
+            gr.Info("NeRFTalk模型导入成功")
+            gr.Warning("NeRFTalk模型是针对单个人进行训练的，内置了奥班马Obama的模型，上传图片无效")
+        except Exception as e:
+            gr.Warning("NeRFTalk模型加载失败", e)
+    else:
+        gr.Warning("未知TFG模型，可提issue和PR 或者 建议更新模型")
+    return model_name
+
+def tts_model_change(model_name, progress=gr.Progress(track_tqdm=True)):
+    global tts
+
+    # 清理显存，在加载新的模型之前释放不必要的显存
+    clear_memory()
+
+    if model_name == 'Edge-TTS':
+        # tts = EdgeTTS()
+        if edgetts.network:
+            gr.Info("EdgeTTS模型导入成功")
+        else:
+            gr.Warning("EdgeTTS模型加载失败，请检查网络是否正常连接，否则无法使用")
+    elif model_name == 'PaddleTTS':
+        try:
+            from TTS import PaddleTTS
+            tts = PaddleTTS()
+            gr.Info("PaddleTTS模型导入成功")
+        except Exception as e:
+            gr.Warning(f"PaddleTTS模型下载失败 {e}")
+    elif model_name == 'GPT-SoVITS克隆声音':
+        try:
+            gpt_path = "GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt"
+            sovits_path = "GPT_SoVITS/pretrained_models/s2G488k.pth"
+            vits.load_model(gpt_path, sovits_path)
+            gr.Info("模型加载成功")
+        except Exception as e:
+            gr.Warning(f"模型加载失败 {e}")
+        gr.Warning("注意注意⚠️：GPT-SoVITS要上传参考音频进行克隆，请点击TTS Method语音方法调节操作")
+    else:
+        gr.Warning("未知TTS模型，可提issue和PR 或者 建议更新模型")
+    return model_name
+
+def success_print(text):
+    print(f"\033[1;32;40m{text}\033[0m")
+
+def error_print(text):
+    print(f"\033[1;31;40m{text}\033[0m")
+
+if __name__ == "__main__":
+    llm_class = LLM(mode='offline')
+    llm = llm_class.init_model('直接回复 Direct Reply')
+    success_print("默认不使用LLM模型，直接回复问题，同时减少显存占用！")
+    
+    try:
+        from VITS import *
+        vits = GPT_SoVITS()
+        success_print("Success!!! GPT-SoVITS模块加载成功，语音克隆默认使用GPT-SoVITS模型")
+        # gpt_path = "GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt"
+        # sovits_path = "GPT_SoVITS/pretrained_models/s2G488k.pth"
+        # vits.load_model(gpt_path, sovits_path)
+    except Exception as e:
+        error_print(f"GPT-SoVITS Error: {e}")
+        error_print("如果使用VITS，请先下载GPT-SoVITS模型和安装环境")
+    
+    try:
+        from TFG import SadTalker
+        talker = SadTalker(lazy_load=True)
+        success_print("Success!!! SadTalker模块加载成功，默认使用SadTalker模型")
+    except Exception as e:
+        error_print(f"SadTalker Error: {e}")
+        error_print("如果使用SadTalker，请先下载SadTalker模型")
+    
+    try:
+        from ASR import WhisperASR
+        if os.path.exists('Whisper/base.pt'):
+            asr = WhisperASR('Whisper/base.pt')
+        else:
+            asr = WhisperASR('base')
+        success_print("Success!!! WhisperASR模块加载成功，默认使用Whisper-base模型")
+    except Exception as e:
+        error_print(f"ASR Error: {e}")
+        error_print("如果使用FunASR，请先下载WhisperASR模型和安装环境")
+    
+    # 判断显存是否8g，若小于8g不建议使用MuseTalk功能
+    # Check if GPU is available and has at least 8GB of memory
+    if torch.cuda.is_available():
+        gpu_memory = torch.cuda.get_device_properties(0).total_memory / (1024 ** 3)  # Convert bytes to GB
+        if gpu_memory < 8:
+            error_print("警告: 您的显卡显存小于8GB，不建议使用MuseTalk功能")
+    
+    try:
+        from TFG import MuseTalk_RealTime
+        musetalker = MuseTalk_RealTime()
+        success_print("Success!!! MuseTalk模块加载成功")
+    except Exception as e:
+        error_print(f"MuseTalk Error: {e}")
+        error_print("如果使用MuseTalk，请先下载MuseTalk模型")
+
+    tts = edgetts
+    if not tts.network:
+        error_print("EdgeTTS模块加载失败，请检查网络是否正常连接，否则无法使用")
+
+    gr.close_all()
+    # demo_app = app()
+    demo_img = app_img()
+    demo_multi = app_multi()
+    # demo_vits = app_vits()
+    # demo_talk = app_talk()
+    demo_muse = app_muse()
+    demo = gr.TabbedInterface(interface_list = [
+        # demo_app, 
+        demo_img, 
+        demo_multi, 
+        # demo_vits, 
+        # demo_talk,
+        demo_muse,
+        ], 
+        tab_names = [
+            "个性化角色互动", 
+            "数字人多轮智能对话", 
+            "MuseTalk数字人实时对话"
+            ],
+        title = "Linly-Talker WebUI")
+    demo.queue()
+    demo.launch(server_name=ip, # 本地端口localhost:127.0.0.1 全局端口转发:"0.0.0.0"
+                server_port=port,
+                # 似乎在Gradio4.0以上版本可以不使用证书也可以进行麦克风对话
+                # ssl_certfile=ssl_certfile,
+                # ssl_keyfile=ssl_keyfile,
+                # ssl_verify=False,
+                # share=True,
+                debug=True,
+                )

ASR/FunASR.py

@@ -0,0 +1,54 @@
+'''
+Reference: https://github.com/alibaba-damo-academy/FunASR
+pip install funasr
+pip install modelscope
+pip install -U rotary_embedding_torch
+'''
+try:
+    from funasr import AutoModel
+except:
+    print("如果想使用FunASR，请先安装funasr，若使用Whisper，请忽略此条信息")
+import os
+import sys
+sys.path.append('./')
+from src.cost_time import calculate_time    
+
+class FunASR:
+    def __init__(self) -> None:
+        # 定义模型的自定义路径
+        model_path = "FunASR/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
+        vad_model_path = "FunASR/speech_fsmn_vad_zh-cn-16k-common-pytorch"
+        punc_model_path = "FunASR/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
+
+        # 检查文件是否存在于 FunASR 目录下
+        model_exists = os.path.exists(model_path)
+        vad_model_exists = os.path.exists(vad_model_path)
+        punc_model_exists = os.path.exists(punc_model_path)
+        # Modelscope AutoDownload
+        self.model = AutoModel(
+            model=model_path if model_exists else "paraformer-zh",
+            vad_model=vad_model_path if vad_model_exists else "fsmn-vad",
+            punc_model=punc_model_path if punc_model_exists else "ct-punc-c",
+        )
+        # 自定义路径
+        # self.model = AutoModel(model="FunASR/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch", # model_revision="v2.0.4",
+        #         vad_model="FunASR/speech_fsmn_vad_zh-cn-16k-common-pytorch", # vad_model_revision="v2.0.4",
+        #         punc_model="FunASR/punc_ct-transformer_zh-cn-common-vocab272727-pytorch", # punc_model_revision="v2.0.4",
+        #         # spk_model="cam++", spk_model_revision="v2.0.2",
+        #         )
+    @calculate_time
+    def transcribe(self, audio_file):
+        res = self.model.generate(input=audio_file, 
+            batch_size_s=300)
+        print(res)
+        return res[0]['text']
+    
+        
+if __name__ == "__main__":
+    import os
+    # 创建ASR对象并进行语音识别
+    audio_file = "output.wav"  # 音频文件路径
+    if not os.path.exists(audio_file):
+        os.system('edge-tts --text "hello" --write-media output.wav')
+    asr = FunASR()
+    print(asr.transcribe(audio_file))

ASR/README.md

@@ -0,0 +1,77 @@
+## ASR 同数字人沟通的桥梁
+
+### Whisper OpenAI
+
+Whisper 是一个自动语音识别 (ASR) 系统，它使用从网络上收集的 680,000 小时多语言和多任务监督数据进行训练。使用如此庞大且多样化的数据集可以提高对口音、背景噪音和技术语言的鲁棒性。此外，它还支持多种语言的转录，以及将这些语言翻译成英语。
+
+使用方法很简单，我们只要安装以下库，后续模型会自动下载
+
+```bash
+pip install -U openai-whisper
+```
+
+借鉴OpenAI的Whisper实现了ASR的语音识别，具体使用方法参考 [https://github.com/openai/whisper](https://github.com/openai/whisper)
+
+```python
+'''
+https://github.com/openai/whisper
+pip install -U openai-whisper
+'''
+import whisper
+
+class WhisperASR:
+    def __init__(self, model_path):
+        self.LANGUAGES = {
+            "en": "english",
+            "zh": "chinese",
+        }
+        self.model = whisper.load_model(model_path)
+        
+    def transcribe(self, audio_file):
+        result = self.model.transcribe(audio_file)
+        return result["text"]
+```
+
+
+
+### FunASR Alibaba
+
+阿里的`FunASR`的语音识别效果也是相当不错，而且时间也是比whisper更快的，更能达到实时的效果，所以也将FunASR添加进去了，在ASR文件夹下的FunASR文件里可以进行体验，参考 [https://github.com/alibaba-damo-academy/FunASR](https://github.com/alibaba-damo-academy/FunASR)
+
+需要注意的是，在第一次运行的时候，需要安装以下库。
+
+```bash
+pip install funasr
+pip install modelscope
+pip install -U rotary_embedding_torch
+```
+
+```python
+'''
+Reference: https://github.com/alibaba-damo-academy/FunASR
+pip install funasr
+pip install modelscope
+pip install -U rotary_embedding_torch
+'''
+try:
+    from funasr import AutoModel
+except:
+    print("如果想使用FunASR，请先安装funasr，若使用Whisper，请忽略此条信息")   
+
+class FunASR:
+    def __init__(self) -> None:
+        self.model = AutoModel(model="paraformer-zh", model_revision="v2.0.4",
+                vad_model="fsmn-vad", vad_model_revision="v2.0.4",
+                punc_model="ct-punc-c", punc_model_revision="v2.0.4",
+                # spk_model="cam++", spk_model_revision="v2.0.2",
+                )
+
+    def transcribe(self, audio_file):
+        res = self.model.generate(input=audio_file, 
+            batch_size_s=300)
+        print(res)
+        return res[0]['text']
+```
+
+
+

ASR/Whisper.py

@@ -0,0 +1,129 @@
+'''
+https://github.com/openai/whisper
+pip install -U openai-whisper
+'''
+import whisper
+import sys
+sys.path.append('./')
+from src.cost_time import calculate_time 
+
+class WhisperASR:
+    def __init__(self, model_path):
+        self.LANGUAGES = {
+            "en": "english",
+            "zh": "chinese",
+            "de": "german",
+            "es": "spanish",
+            "ru": "russian",
+            "ko": "korean",
+            "fr": "french",
+            "ja": "japanese",
+            "pt": "portuguese",
+            "tr": "turkish",
+            "pl": "polish",
+            "ca": "catalan",
+            "nl": "dutch",
+            "ar": "arabic",
+            "sv": "swedish",
+            "it": "italian",
+            "id": "indonesian",
+            "hi": "hindi",
+            "fi": "finnish",
+            "vi": "vietnamese",
+            "he": "hebrew",
+            "uk": "ukrainian",
+            "el": "greek",
+            "ms": "malay",
+            "cs": "czech",
+            "ro": "romanian",
+            "da": "danish",
+            "hu": "hungarian",
+            "ta": "tamil",
+            "no": "norwegian",
+            "th": "thai",
+            "ur": "urdu",
+            "hr": "croatian",
+            "bg": "bulgarian",
+            "lt": "lithuanian",
+            "la": "latin",
+            "mi": "maori",
+            "ml": "malayalam",
+            "cy": "welsh",
+            "sk": "slovak",
+            "te": "telugu",
+            "fa": "persian",
+            "lv": "latvian",
+            "bn": "bengali",
+            "sr": "serbian",
+            "az": "azerbaijani",
+            "sl": "slovenian",
+            "kn": "kannada",
+            "et": "estonian",
+            "mk": "macedonian",
+            "br": "breton",
+            "eu": "basque",
+            "is": "icelandic",
+            "hy": "armenian",
+            "ne": "nepali",
+            "mn": "mongolian",
+            "bs": "bosnian",
+            "kk": "kazakh",
+            "sq": "albanian",
+            "sw": "swahili",
+            "gl": "galician",
+            "mr": "marathi",
+            "pa": "punjabi",
+            "si": "sinhala",
+            "km": "khmer",
+            "sn": "shona",
+            "yo": "yoruba",
+            "so": "somali",
+            "af": "afrikaans",
+            "oc": "occitan",
+            "ka": "georgian",
+            "be": "belarusian",
+            "tg": "tajik",
+            "sd": "sindhi",
+            "gu": "gujarati",
+            "am": "amharic",
+            "yi": "yiddish",
+            "lo": "lao",
+            "uz": "uzbek",
+            "fo": "faroese",
+            "ht": "haitian creole",
+            "ps": "pashto",
+            "tk": "turkmen",
+            "nn": "nynorsk",
+            "mt": "maltese",
+            "sa": "sanskrit",
+            "lb": "luxembourgish",
+            "my": "myanmar",
+            "bo": "tibetan",
+            "tl": "tagalog",
+            "mg": "malagasy",
+            "as": "assamese",
+            "tt": "tatar",
+            "haw": "hawaiian",
+            "ln": "lingala",
+            "ha": "hausa",
+            "ba": "bashkir",
+            "jw": "javanese",
+            "su": "sundanese",
+        }
+        self.model = whisper.load_model(model_path)
+
+    @calculate_time
+    def transcribe(self, audio_file):
+        result = self.model.transcribe(audio_file)
+        return result["text"]
+
+
+if __name__ == "__main__":
+    import os
+    # 创建ASR对象并进行语音识别
+    model_path = "./Whisper/tiny.pt"  # 模型路径
+    audio_file = "output.wav"  # 音频文件路径
+    if not os.path.exists(audio_file):
+        os.system('edge-tts --text "hello" --write-media output.wav')
+    asr = WhisperASR(model_path)
+    print(asr.transcribe(audio_file))

ASR/__init__.py

@@ -0,0 +1,4 @@
+from .Whisper import WhisperASR
+from .FunASR import FunASR
+
+__all__ = ['WhisperASR', 'FunASR']

ASR/requirements_funasr.txt

@@ -0,0 +1,3 @@
+funasr
+modelscope
+# rotary_embedding_torch

AutoDL部署.md

@@ -0,0 +1,234 @@
+# 在AutoDL平台部署Linly-Talker (0基础小白超详细教程)
+
+<!-- TOC -->
+
+- [在AutoDL平台部署Linly-Talker 0基础小白超详细教程](#%E5%9C%A8autodl%E5%B9%B3%E5%8F%B0%E9%83%A8%E7%BD%B2linly-talker-0%E5%9F%BA%E7%A1%80%E5%B0%8F%E7%99%BD%E8%B6%85%E8%AF%A6%E7%BB%86%E6%95%99%E7%A8%8B)
+    - [快速上手直接使用镜像以下安装操作全免](#%E5%BF%AB%E9%80%9F%E4%B8%8A%E6%89%8B%E7%9B%B4%E6%8E%A5%E4%BD%BF%E7%94%A8%E9%95%9C%E5%83%8F%E4%BB%A5%E4%B8%8B%E5%AE%89%E8%A3%85%E6%93%8D%E4%BD%9C%E5%85%A8%E5%85%8D)
+    - [一、注册AutoDL](#%E4%B8%80%E6%B3%A8%E5%86%8Cautodl)
+    - [二、创建实例](#%E4%BA%8C%E5%88%9B%E5%BB%BA%E5%AE%9E%E4%BE%8B)
+        - [登录AutoDL，进入算力市场，选择机器](#%E7%99%BB%E5%BD%95autodl%E8%BF%9B%E5%85%A5%E7%AE%97%E5%8A%9B%E5%B8%82%E5%9C%BA%E9%80%89%E6%8B%A9%E6%9C%BA%E5%99%A8)
+        - [配置基础镜像](#%E9%85%8D%E7%BD%AE%E5%9F%BA%E7%A1%80%E9%95%9C%E5%83%8F)
+        - [无卡模式开机](#%E6%97%A0%E5%8D%A1%E6%A8%A1%E5%BC%8F%E5%BC%80%E6%9C%BA)
+    - [三、部署环境](#%E4%B8%89%E9%83%A8%E7%BD%B2%E7%8E%AF%E5%A2%83)
+        - [进入终端](#%E8%BF%9B%E5%85%A5%E7%BB%88%E7%AB%AF)
+        - [下载代码文件](#%E4%B8%8B%E8%BD%BD%E4%BB%A3%E7%A0%81%E6%96%87%E4%BB%B6)
+        - [下载模型文件](#%E4%B8%8B%E8%BD%BD%E6%A8%A1%E5%9E%8B%E6%96%87%E4%BB%B6)
+    - [四、Linly-Talker项目](#%E5%9B%9Blinly-talker%E9%A1%B9%E7%9B%AE)
+        - [环境安装](#%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85)
+        - [端口设置](#%E7%AB%AF%E5%8F%A3%E8%AE%BE%E7%BD%AE)
+        - [有卡开机](#%E6%9C%89%E5%8D%A1%E5%BC%80%E6%9C%BA)
+        - [运行网页版对话webui](#%E8%BF%90%E8%A1%8C%E7%BD%91%E9%A1%B5%E7%89%88%E5%AF%B9%E8%AF%9Dwebui)
+        - [端口映射](#%E7%AB%AF%E5%8F%A3%E6%98%A0%E5%B0%84)
+        - [体验Linly-Talker（成功）](#%E4%BD%93%E9%AA%8Clinly-talker%E6%88%90%E5%8A%9F)
+
+<!-- /TOC -->
+
+
+
+## 快速上手直接使用镜像(以下安装操作全免)
+
+若使用我设定好的镜像，可以直接运行即可，不需要安装环境，直接运行webui.py或者是app_talk.py即可体验，不需要安装任何环境，可直接跳到4.4即可
+
+访问后在自定义设置里面打开端口，默认是6006端口，直接使用运行即可！
+
+```bash
+python webui.py
+python app_talk.py
+```
+
+环境模型都安装好了，直接使用即可，镜像地址在：[https://www.codewithgpu.com/i/Kedreamix/Linly-Talker/Kedreamix-Linly-Talker](https://www.codewithgpu.com/i/Kedreamix/Linly-Talker/Kedreamix-Linly-Talker)，感谢大家的支持
+
+
+
+## 一、注册AutoDL
+
+[AutoDL官网](https://www.autodl.com/home) 注册账户好并充值，自己选择机器，我觉得如果正常跑一下，5元已经够了
+
+![注册AutoDL](https://pic1.zhimg.com/v2-f56bc692a0d22fb1ae749b7697ff5d0f.png)
+
+## 二、创建实例
+
+### 2.1 登录AutoDL，进入算力市场，选择机器
+
+这一部分实际上我觉得12g都OK的，无非是速度问题而已
+
+![选择RTX 3090机器](https://picx.zhimg.com/v2-824956d591eead5d3ed4de87c59258a6.png)
+
+
+
+### 2.2 配置基础镜像
+
+选择镜像，最好选择2.0以上可以体验克隆声音功能，其他无所谓
+
+![配置基础镜像](https://pic1.zhimg.com/v2-8d064d809e15673dc6f2be8f2ef83ae7.png)
+
+
+
+### 2.3 无卡模式开机
+
+创建成功后为了省钱先关机，然后使用无卡模式开机。
+无卡模式一个小时只需要0.1元，比较适合部署环境。
+
+![无卡模式开机](https://picx.zhimg.com/v2-118eead549c35ad06d946b00cd93c668.png)
+
+## 三、部署环境
+
+### 3.1 进入终端
+
+打开jupyterLab，进入数据盘（autodl-tmp），打开终端，将Linly-Talker模型下载到数据盘中。
+
+![进入终端](https://pica.zhimg.com/v2-7c29a3f68c04b9af0963d4acaa63f59c.png)
+
+
+
+### 3.2 下载代码文件
+
+根据Github上的说明，使用命令行下载模型文件和代码文件，利用学术加速会快一点
+
+```bash
+# 开启学术镜像，更快的clone代码 参考 https://www.autodl.com/docs/network_turbo/
+source /etc/network_turbo
+
+cd /root/autodl-tmp/
+# 下载代码
+git clone https://github.com/Kedreamix/Linly-Talker.git
+
+# 取消学术加速
+unset http_proxy && unset https_proxy
+```
+
+
+
+### 3.3 下载模型文件
+
+安装git lfs
+
+```sh
+curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
+sudo apt-get install git-lfs
+```
+
+![安装git lfs](https://pic1.zhimg.com/v2-084b5f048d1a70d8712bd6a6891c673c.png)
+
+根据 [https://www.modelscope.cn/Kedreamix/Linly-Talker](https://www.modelscope.cn/Kedreamix/Linly-Talker) 下载模型文件，走modelscope还是很快的，不过文件有点多，还是得等一下，记住是在Linly-Talker代码路径下执行这个文件
+
+```bash
+cd /root/autodl-tmp/Linly-Talker/
+git lfs install
+git lfs clone https://www.modelscope.cn/Kedreamix/Linly-Talker.git
+```
+
+![下载模型文件](https://picx.zhimg.com/v2-1f61be9be69053ebe16c72762c350d18.png)
+
+等待一段时间下载完以后，利用命令将模型移动到指定目���，直接复制即可
+
+```bash
+# 移动所有模型到当前目录
+# checkpoint中含有SadTalker和Wav2Lip
+mv Linly-Talker/checkpoints/* ./checkpoints
+
+# SadTalker的增强GFPGAN
+# pip install gfpgan
+# mv Linly-Talker/gfpan ./
+
+# 语音克隆模型
+mv Linly-Talker/GPT_SoVITS/pretrained_models/* ./GPT_SoVITS/pretrained_models/
+
+# Qwen大模型
+mv Linly-Talker/Qwen ./
+```
+
+
+
+## 四、Linly-Talker项目
+
+### 4.1 环境安装
+
+进入代码路径，进行安装环境，由于选了镜像是含有pytorch的，所以只需要进行安装其他依赖即可
+
+```bash
+cd /root/autodl-tmp/Linly-Talker
+
+conda install -q ffmpeg # ffmpeg==4.2.2
+
+# 安装Linly-Talker对应依赖
+pip install -r requirements_app.txt
+
+# 安装语音克隆对应的依赖
+pip install -r VITS/requirements_gptsovits.txt
+```
+
+
+
+### 4.2 端口设置
+
+由于似乎autodl开放的是6006端口，所以这里面的端口映射也可以改一下成6006，这里吗只需要修改configs.py文件里面的port为6006即可
+
+![端口设置](https://picx.zhimg.com/v2-d9516998de3e6ffea282c4c6776c126f.png)
+
+除此之外，我发现其实对于autodl来说，不是很支持https的端口映射，所以需要注释掉几行代码即可，在webui.py的最后几行注释掉代码ssl相关代码
+
+```bash
+    demo.launch(server_name="127.0.0.1", # 本地端口localhost:127.0.0.1 全局端口转发:"0.0.0.0"
+                server_port=port,
+                # 似乎在Gradio4.0以上版本可以不使用证书也可以进行麦克风对话
+                # ssl_certfile=ssl_certfile,
+                # ssl_keyfile=ssl_keyfile,
+                # ssl_verify=False,
+                debug=True,
+                )
+```
+
+如果使用app.py同理
+
+![http设置](https://picx.zhimg.com/v2-fd4aed3c765b8a4a425e12520a1461ae.png)
+
+### 4.3 有卡开机
+
+进入autodl容器实例界面，执行关机操作，然后进行有卡开机，开机后打开jupyterLab。
+
+查看配置
+
+```bash
+nvidia-smi
+```
+
+![有卡开机](https://pic1.zhimg.com/v2-baff41e6634fd25fb5d39806b86b1c40.png)
+
+
+
+### 4.4 运行网页版对话webui
+
+需要有卡模式开机，执行下边命令，这里面就跟代码是一模一样的了
+
+```bash
+python webui.py
+```
+
+![运行网页版对话webui](https://pic1.zhimg.com/v2-c1c8a2653cef9d4b044fb85555411928.png)
+
+
+
+### 4.4 端口映射
+
+这可以直接打开autodl的自定义服务，默认是6006端口，我们已经设置了，所以直接使用即可
+
+![端口映射](https://picx.zhimg.com/v2-19a51c44478a1fad9e0de593ee4d547f.png)
+
+
+
+### 4.5 体验Linly-Talker（成功）
+
+点开网页，即可正确执行Linly-Talker，这一部分就跟视频一模一样了
+
+![体验Linly-Talker](https://picx.zhimg.com/v2-1b67099f2903fdcac732b784477851aa.png)
+
+
+
+
+
+ssh端口映射工具：windows：[https://autodl-public.ks3-cn-beijing.ksyuncs.com/tool/AutoDL-SSH-Tools.zip](https://autodl-public.ks3-cn-beijing.ksyuncs.com/tool/AutoDL-SSH-Tools.zip)
+
+**！！！注意：不用了，一定要去控制台=》容器实例，把镜像实例关机，它是按时收费的，不关机会一直扣费的。**
+
+**建议选北京区的，稍微便宜一些。可以晚上部署，网速快，便宜的GPU也充足。白天部署，北京区的GPU容易没有。**

FunASR/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/README.md

@@ -0,0 +1,272 @@
+---
+tasks:
+- punctuation
+domain:
+- audio
+model-type:
+- Classification
+frameworks:
+- pytorch
+metrics:
+- f1_score
+license: Apache License 2.0
+language: 
+- cn
+tags:
+- FunASR
+- CT-Transformer
+- Alibaba
+- ICASSP 2020
+datasets:
+  train:
+  - 33M-samples online data
+  test:
+  - wikipedia data test
+  - 10000 industrial Mandarin sentences test
+widgets:
+  - task: punctuation
+    model_revision: v2.0.4
+    inputs:
+      - type: text
+        name: input
+        title: 文本
+    examples:
+      - name: 1
+        title: 示例1
+        inputs:
+          - name: input
+            data: 我们都是木头人不会讲话不会动
+    inferencespec:
+      cpu: 1 #CPU数量
+      memory: 4096
+---
+
+# Controllable Time-delay Transformer模型介绍
+
+[//]: # (Controllable Time-delay Transformer 模型是一种端到端标点分类模型。)
+
+[//]: # (常规的Transformer会依赖很远的未来信息，导致长时间结果不固定。Controllable Time-delay Transformer 在效果无损的情况下，有效控制标点的延时。)
+
+# Highlights
+- 中文标点通用模型：可用于语音识别模型输出文本的标点预测。
+  - 基于[Paraformer-large长音频模型](https://www.modelscope.cn/models/damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary)场景的使用
+  - 基于[FunASR框架](https://github.com/alibaba-damo-academy/FunASR)，可进行ASR，VAD，标点的自由组合
+  - 基于纯文本输入的标点预测
+
+## <strong>[FunASR开源项目介绍](https://github.com/alibaba-damo-academy/FunASR)</strong>
+<strong>[FunASR](https://github.com/alibaba-damo-academy/FunASR)</strong>希望在语音识别的学术研究和工业应用之间架起一座桥梁。通过发布工业级语音识别模型的训练和微调，研究人员和开发人员可以更方便地进行语音识别模型的研究和生产，并推动语音识别生态的发展。让语音识别更有趣！
+
+[**github仓库**](https://github.com/alibaba-damo-academy/FunASR)
+| [**最新动态**](https://github.com/alibaba-damo-academy/FunASR#whats-new) 
+| [**环境安装**](https://github.com/alibaba-damo-academy/FunASR#installation)
+| [**服务部署**](https://www.funasr.com)
+| [**模型库**](https://github.com/alibaba-damo-academy/FunASR/tree/main/model_zoo)
+| [**联系我们**](https://github.com/alibaba-damo-academy/FunASR#contact)
+
+
+## 模型原理介绍
+
+Controllable Time-delay Transformer是达摩院语音团队提出的高效后处理框架中的标点模块。本项目为中文通用标点模型，模型可以被应用于文本类输入的标点预测，也可应用于语音识别结果的后处理步骤，协助语音识别模块输出具有可读性的文本结果。
+
+<p align="center">
+<img src="fig/struct.png" alt="Controllable Time-delay Transformer模型结构"  width="500" />
+
+Controllable Time-delay Transformer 模型结构如上图所示，由 Embedding、Encoder 和 Predictor 三部分组成。Embedding 是词向量叠加位置向量。Encoder可以采用不同的网络结构，例如self-attention，conformer，SAN-M等。Predictor 预测每个token后的标点类型。
+
+在模型的选择上采用了性能优越的Transformer模型。Transformer模型在获得良好性能的同时，由于模型自身序列化输入等特性，会给系统带来较大时延。常规的Transformer可以看到未来的全部信息，导致标点会依赖很远的未来信息。这会给用户带来一种标点一直在变化刷新，长时间结果不固定的不良感受。基于这一问题，我们创新性的提出了可控时延的Transformer模型（Controllable Time-Delay Transformer, CT-Transformer），在模型性能无损失的情况下，有效控制标点的延时。
+
+更详细的细节见：
+- 论文： [CONTROLLABLE TIME-DELAY TRANSFORMER FOR REAL-TIME PUNCTUATION PREDICTION AND DISFLUENCY DETECTION](https://arxiv.org/pdf/2003.01309.pdf)
+
+## 基于ModelScope进行推理
+
+以下为三种支持格式及api调用方式参考如下范例：
+- text.scp文件路径，例如example/punc_example.txt，格式为： key + "\t" + value
+```sh
+cat example/punc_example.txt
+1       跨境河流是养育沿岸人民的生命之源
+2       从存储上来说仅仅是全景图片它就会是图片的四倍的容量
+3       那今天的会就到这里吧happy new year明年见
+```
+```python
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+inference_pipline = pipeline(
+    task=Tasks.punctuation,
+    model='damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch',
+    model_revision="v2.0.4")
+
+rec_result = inference_pipline(input='example/punc_example.txt')
+print(rec_result)
+```
+- text二进制数据，例如：用户直接从文件里读出bytes数据
+```python
+rec_result = inference_pipline(input='我们都是木头人不会讲话不会动')
+```
+- text文件url，例如：https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_text/punc_example.txt
+```python
+rec_result = inference_pipline(input='https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_text/punc_example.txt')
+```
+
+
+## 基于FunASR进行推理
+
+下面为快速上手教程，测试音频（[中文](https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.wav)，[英文](https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_en.wav)）
+
+### 可执行命令行
+在命令行终端执行：
+
+```shell
+funasr ++model=paraformer-zh ++vad_model="fsmn-vad" ++punc_model="ct-punc" ++input=vad_example.wav
+```
+
+注：支持单条音频文件识别，也支持文件列表，列表为kaldi风格wav.scp：`wav_id   wav_path`
+
+### python示例
+#### 非实时语音识别
+```python
+from funasr import AutoModel
+# paraformer-zh is a multi-functional asr model
+# use vad, punc, spk or not as you need
+model = AutoModel(model="paraformer-zh", model_revision="v2.0.4",
+                  vad_model="fsmn-vad", vad_model_revision="v2.0.4",
+                  punc_model="ct-punc-c", punc_model_revision="v2.0.4",
+                  # spk_model="cam++", spk_model_revision="v2.0.2",
+                  )
+res = model.generate(input=f"{model.model_path}/example/asr_example.wav", 
+            batch_size_s=300, 
+            hotword='魔搭')
+print(res)
+```
+注：`model_hub`：表示模型仓库，`ms`为选择modelscope下载，`hf`为选择huggingface下载。
+
+#### 实时语音识别
+
+```python
+from funasr import AutoModel
+
+chunk_size = [0, 10, 5] #[0, 10, 5] 600ms, [0, 8, 4] 480ms
+encoder_chunk_look_back = 4 #number of chunks to lookback for encoder self-attention
+decoder_chunk_look_back = 1 #number of encoder chunks to lookback for decoder cross-attention
+
+model = AutoModel(model="paraformer-zh-streaming", model_revision="v2.0.4")
+
+import soundfile
+import os
+
+wav_file = os.path.join(model.model_path, "example/asr_example.wav")
+speech, sample_rate = soundfile.read(wav_file)
+chunk_stride = chunk_size[1] * 960 # 600ms
+
+cache = {}
+total_chunk_num = int(len((speech)-1)/chunk_stride+1)
+for i in range(total_chunk_num):
+    speech_chunk = speech[i*chunk_stride:(i+1)*chunk_stride]
+    is_final = i == total_chunk_num - 1
+    res = model.generate(input=speech_chunk, cache=cache, is_final=is_final, chunk_size=chunk_size, encoder_chunk_look_back=encoder_chunk_look_back, decoder_chunk_look_back=decoder_chunk_look_back)
+    print(res)
+```
+
+注：`chunk_size`为流式延时配置，`[0,10,5]`表示上屏实时出字粒度为`10*60=600ms`，未来信息为`5*60=300ms`。每次推理输入为`600ms`（采样点数为`16000*0.6=960`），输出为对应文字，最后一个语音片段输入需要设置`is_final=True`来强制输出最后一个字。
+
+#### 语音端点检测（非实时）
+```python
+from funasr import AutoModel
+
+model = AutoModel(model="fsmn-vad", model_revision="v2.0.4")
+
+wav_file = f"{model.model_path}/example/asr_example.wav"
+res = model.generate(input=wav_file)
+print(res)
+```
+
+#### 语音端点检测（实时）
+```python
+from funasr import AutoModel
+
+chunk_size = 200 # ms
+model = AutoModel(model="fsmn-vad", model_revision="v2.0.4")
+
+import soundfile
+
+wav_file = f"{model.model_path}/example/vad_example.wav"
+speech, sample_rate = soundfile.read(wav_file)
+chunk_stride = int(chunk_size * sample_rate / 1000)
+
+cache = {}
+total_chunk_num = int(len((speech)-1)/chunk_stride+1)
+for i in range(total_chunk_num):
+    speech_chunk = speech[i*chunk_stride:(i+1)*chunk_stride]
+    is_final = i == total_chunk_num - 1
+    res = model.generate(input=speech_chunk, cache=cache, is_final=is_final, chunk_size=chunk_size)
+    if len(res[0]["value"]):
+        print(res)
+```
+
+#### 标点恢复
+```python
+from funasr import AutoModel
+
+model = AutoModel(model="ct-punc", model_revision="v2.0.4")
+
+res = model.generate(input="那今天的会就到这里吧 happy new year 明年见")
+print(res)
+```
+
+#### 时间戳预测
+```python
+from funasr import AutoModel
+
+model = AutoModel(model="fa-zh", model_revision="v2.0.4")
+
+wav_file = f"{model.model_path}/example/asr_example.wav"
+text_file = f"{model.model_path}/example/text.txt"
+res = model.generate(input=(wav_file, text_file), data_type=("sound", "text"))
+print(res)
+```
+
+更多详细用法（[示例](https://github.com/alibaba-damo-academy/FunASR/tree/main/examples/industrial_data_pretraining)）
+
+
+## 微调
+
+详细用法（[示例](https://github.com/alibaba-damo-academy/FunASR/tree/main/examples/industrial_data_pretraining)）
+
+
+
+
+
+## Benchmark
+中文标点预测通用模型在自采集的通用领域业务场景数据上有良好效果。训练数据大约33M个sample，每个sample可能包含1句或多句。
+
+### 自采集数据（20000+ samples）
+
+| precision                            | recall                                | f1_score                              |
+|:------------------------------------:|:-------------------------------------:|:-------------------------------------:|
+| <div style="width: 150pt">53.8</div> | <div style="width: 150pt">60.0</div>  | <div style="width: 150pt">56.5</div>  | 
+
+## 使用方式以及适用范围
+
+运行范围
+- 支持Linux-x86_64、Mac和Windows运行。
+
+使用方式
+- 直接推理：可以直接对输入文本进行计算，输出带有标点的目标文字。
+
+使用范围与目标场景
+- 适合对文本数据进行标点预测，文本长度不限。
+
+## 相关论文以及引用信息
+
+```BibTeX
+@inproceedings{chen2020controllable,
+  title={Controllable Time-Delay Transformer for Real-Time Punctuation Prediction and Disfluency Detection},
+  author={Chen, Qian and Chen, Mengzhe and Li, Bo and Wang, Wen},
+  booktitle={ICASSP 2020-2020 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
+  pages={8069--8073},
+  year={2020},
+  organization={IEEE}
+}
+```
+

FunASR/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/config.yaml

@@ -0,0 +1,46 @@
+model: CTTransformer
+model_conf:
+    ignore_id: 0
+    embed_unit: 256
+    att_unit: 256
+    dropout_rate: 0.1
+    punc_list:
+        - <unk>
+        - _
+        - ，
+        - 。
+        - ？
+        - 、
+    punc_weight:
+        - 1.0
+        - 1.0
+        - 1.0
+        - 1.0
+        - 1.0
+        - 1.0
+    sentence_end_id: 3
+
+encoder: SANMEncoder
+encoder_conf:
+    input_size: 256
+    output_size: 256
+    attention_heads: 8
+    linear_units: 1024
+    num_blocks: 4
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.0
+    input_layer: pe
+    pos_enc_class: SinusoidalPositionEncoder
+    normalize_before: true
+    kernel_size: 11
+    sanm_shfit: 0
+    selfattention_layer_type: sanm
+    padding_idx: 0
+
+tokenizer: CharTokenizer
+tokenizer_conf:
+  unk_symbol: <unk>
+
+
+

FunASR/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/configuration.json

@@ -0,0 +1,13 @@
+{
+  "framework": "pytorch",
+  "task" : "punctuation",
+  "model": {"type" : "funasr"},
+  "pipeline": {"type":"funasr-pipeline"},
+  "model_name_in_hub": {
+    "ms":"iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch", 
+    "hf":""},
+  "file_path_metas": {
+    "init_param":"model.pt", 
+    "config":"config.yaml",
+    "tokenizer_conf": {"token_list": "tokens.json"}}
+}

FunASR/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/example/punc_example.txt

@@ -0,0 +1,3 @@
+1	跨境河流是养育沿岸人民的生命之源长期以来为帮助下游地区防灾减灾中方技术人员在上游地区极为恶劣的自然条件下克服巨大困难甚至冒着生命危险向印方提供汛期水文资料处理紧急事件中方重视印方在跨境河流问题上的关切愿意进一步完善双方联合工作机制凡是中方能做的我们都会去做而且会做得更好我请印度朋友们放心中国在上游的任何开发利用都会经过科学规划和论证兼顾上下游的利益
+2	从存储上来说仅仅是全景图片它就会是图片的四倍的容量然后全景的视频会是普通视频八倍的这个存储的容要求而三d的模型会是图片的十倍这都对我们今天运行在的云计算的平台存储的平台提出了更高的要求
+3	那今天的会就到这里吧 happy new year 明年见

FunASR/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5818bb9d933805a916eebe41eb41648f7f9caad30b4bd59d56f3ca135421916
+size 291979892

FunASR/speech_fsmn_vad_zh-cn-16k-common-pytorch/README.md

@@ -0,0 +1,296 @@
+---
+tasks:
+- voice-activity-detection
+domain:
+- audio
+model-type:
+- VAD model
+frameworks:
+- pytorch
+backbone:
+- fsmn
+metrics:
+- f1_score
+license: Apache License 2.0
+language: 
+- cn
+tags:
+- FunASR
+- FSMN
+- Alibaba
+- Online
+datasets:
+  train:
+  - 20,000 hour industrial Mandarin task
+  test:
+  - 20,000 hour industrial Mandarin task
+widgets:
+  - task: voice-activity-detection
+    model_revision: v2.0.4
+    inputs:
+      - type: audio
+        name: input
+        title: 音频
+    examples:
+      - name: 1
+        title: 示例1
+        inputs:
+          - name: input
+            data: git://example/vad_example.wav 
+    inferencespec:
+      cpu: 1 #CPU数量
+      memory: 4096
+---
+
+# FSMN-Monophone VAD 模型介绍
+
+[//]: # (FSMN-Monophone VAD 模型)
+
+## Highlight
+- 16k中文通用VAD模型：可用于检测长语音片段中有效语音的起止时间点。
+  - 基于[Paraformer-large长音频模型](https://www.modelscope.cn/models/damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary)场景的使用
+  - 基于[FunASR框架](https://github.com/alibaba-damo-academy/FunASR)，可进行ASR，VAD，[中文标点](https://www.modelscope.cn/models/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/summary)的自由组合
+  - 基于音频数据的有效语音片段起止时间点检测
+
+## <strong>[FunASR开源项目介绍](https://github.com/alibaba-damo-academy/FunASR)</strong>
+<strong>[FunASR](https://github.com/alibaba-damo-academy/FunASR)</strong>希望在语音识别的学术研究和工业应用之间架起一座桥梁。通过发布工业级语音识别模型的训练和微调，研究人员和开发人员可以更方便地进行语音识别模型的研究和生产，并推动语音识别生态的发展。让语音识别更有趣！
+
+[**github仓库**](https://github.com/alibaba-damo-academy/FunASR)
+| [**最新动态**](https://github.com/alibaba-damo-academy/FunASR#whats-new) 
+| [**环境安装**](https://github.com/alibaba-damo-academy/FunASR#installation)
+| [**服务部署**](https://www.funasr.com)
+| [**模型库**](https://github.com/alibaba-damo-academy/FunASR/tree/main/model_zoo)
+| [**联系我们**](https://github.com/alibaba-damo-academy/FunASR#contact)
+
+
+## 模型原理介绍
+
+FSMN-Monophone VAD是达摩院语音团队提出的高效语音端点检测模型，用于检测输入音频中有效语音的起止时间点信息，并将检测出来的有效音频片段输入识别引擎进行识别，减少无效语音带来的识别错误。
+
+<p align="center">
+<img src="fig/struct.png" alt="VAD模型结构"  width="500" />
+
+FSMN-Monophone VAD模型结构如上图所示：模型结构层面，FSMN模型结构建模时可考虑上下文信息，训练和推理速度快，且时延可控；同时根据VAD模型size以及低时延的要求，对FSMN的网络结构、右看帧数进行了适配。在建模单元层面，speech信息比较丰富，仅用单类来表征学习能力有限，我们将单一speech类升级为Monophone。建模单元细分，可以避免参数平均，抽象学习能力增强，区分性更好。
+
+## 基于ModelScope进行推理
+
+- 推理支持音频格式如下：
+  - wav文件路径，例如：data/test/audios/vad_example.wav
+  - wav文件url，例如：https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.wav
+  - wav二进制数据，格式bytes，例如：用户直接从文件里读出bytes数据或者是麦克风录出bytes数据。
+  - 已解析的audio音频，例如：audio, rate = soundfile.read("vad_example_zh.wav")，类型为numpy.ndarray或者torch.Tensor。
+  - wav.scp文件，需符合如下要求：
+
+```sh
+cat wav.scp
+vad_example1  data/test/audios/vad_example1.wav
+vad_example2  data/test/audios/vad_example2.wav
+...
+```
+
+- 若输入格式wav文件url，api调用方式可参考如下范例：
+
+```python
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+inference_pipeline = pipeline(
+    task=Tasks.voice_activity_detection,
+    model='iic/speech_fsmn_vad_zh-cn-16k-common-pytorch',
+    model_revision="v2.0.4",
+)
+
+segments_result = inference_pipeline(input='https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.wav')
+print(segments_result)
+```
+
+- 输入音频为pcm格式，调用api时需要传入音频采样率参数fs，例如：
+
+```python
+segments_result = inference_pipeline(input='https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.pcm', fs=16000)
+```
+
+- 若输入格式为文件wav.scp(注：文件名需要以.scp结尾)，可添加 output_dir 参数将识别结果写入文件中，参考示例如下：
+
+```python
+inference_pipeline(input="wav.scp", output_dir='./output_dir')
+```
+识别结果输出路径结构如下：
+
+```sh
+tree output_dir/
+output_dir/
+└── 1best_recog
+    └── text
+
+1 directory, 1 files
+```
+text：VAD检测语音起止时间点结果文件（单位：ms）
+
+- 若输入音频为已解析的audio音频，api调用方式可参考如下范例：
+
+```python
+import soundfile
+
+waveform, sample_rate = soundfile.read("vad_example_zh.wav")
+segments_result = inference_pipeline(input=waveform)
+print(segments_result)
+```
+
+- VAD常用参数调整说明（参考：vad.yaml文件）：
+  - max_end_silence_time：尾部连续检测到多长时间静音进行尾点判停，参数范围500ms～6000ms，默认值800ms(该值过低容易出现语音提前截断的情况)。
+  - speech_noise_thres：speech的得分减去noise的得分大于此值则判断为speech，参数范围：（-1,1）
+    - 取值越趋于-1，噪音被误判定为语音的概率越大，FA越高
+    - 取值越趋于+1，语音被误判定为噪音的概率越大，Pmiss越高
+    - 通常情况下，该值会根据当前模型在长语音测试集上的效果取balance
+    
+
+
+
+## 基于FunASR进行推理
+
+下面为快速上手教程，测试音频（[中文](https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.wav)，[英文](https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_en.wav)）
+
+### 可执行命令行
+在命令行终端执行：
+
+```shell
+funasr ++model=paraformer-zh ++vad_model="fsmn-vad" ++punc_model="ct-punc" ++input=vad_example.wav
+```
+
+注：支持单条音频文件识别，也支持文件列表，列表为kaldi风格wav.scp：`wav_id   wav_path`
+
+### python示例
+#### 非实时语音识别
+```python
+from funasr import AutoModel
+# paraformer-zh is a multi-functional asr model
+# use vad, punc, spk or not as you need
+model = AutoModel(model="paraformer-zh", model_revision="v2.0.4",
+                  vad_model="fsmn-vad", vad_model_revision="v2.0.4",
+                  punc_model="ct-punc-c", punc_model_revision="v2.0.4",
+                  # spk_model="cam++", spk_model_revision="v2.0.2",
+                  )
+res = model.generate(input=f"{model.model_path}/example/asr_example.wav", 
+            batch_size_s=300, 
+            hotword='魔搭')
+print(res)
+```
+注：`model_hub`：表示模型仓库，`ms`为选择modelscope下载，`hf`为选择huggingface下载。
+
+#### 实时语音识别
+
+```python
+from funasr import AutoModel
+
+chunk_size = [0, 10, 5] #[0, 10, 5] 600ms, [0, 8, 4] 480ms
+encoder_chunk_look_back = 4 #number of chunks to lookback for encoder self-attention
+decoder_chunk_look_back = 1 #number of encoder chunks to lookback for decoder cross-attention
+
+model = AutoModel(model="paraformer-zh-streaming", model_revision="v2.0.4")
+
+import soundfile
+import os
+
+wav_file = os.path.join(model.model_path, "example/asr_example.wav")
+speech, sample_rate = soundfile.read(wav_file)
+chunk_stride = chunk_size[1] * 960 # 600ms
+
+cache = {}
+total_chunk_num = int(len((speech)-1)/chunk_stride+1)
+for i in range(total_chunk_num):
+    speech_chunk = speech[i*chunk_stride:(i+1)*chunk_stride]
+    is_final = i == total_chunk_num - 1
+    res = model.generate(input=speech_chunk, cache=cache, is_final=is_final, chunk_size=chunk_size, encoder_chunk_look_back=encoder_chunk_look_back, decoder_chunk_look_back=decoder_chunk_look_back)
+    print(res)
+```
+
+注：`chunk_size`为流式延时配置，`[0,10,5]`表示上屏实时出字粒度为`10*60=600ms`，未来信息为`5*60=300ms`。每次推理输入为`600ms`（采样点数为`16000*0.6=960`），输出为对应文字，最后一个语音片段输入需要设置`is_final=True`来强制输出最后一个字。
+
+#### 语音端点检测（非实时）
+```python
+from funasr import AutoModel
+
+model = AutoModel(model="fsmn-vad", model_revision="v2.0.4")
+
+wav_file = f"{model.model_path}/example/asr_example.wav"
+res = model.generate(input=wav_file)
+print(res)
+```
+
+#### 语音端点检测（实时）
+```python
+from funasr import AutoModel
+
+chunk_size = 200 # ms
+model = AutoModel(model="fsmn-vad", model_revision="v2.0.4")
+
+import soundfile
+
+wav_file = f"{model.model_path}/example/vad_example.wav"
+speech, sample_rate = soundfile.read(wav_file)
+chunk_stride = int(chunk_size * sample_rate / 1000)
+
+cache = {}
+total_chunk_num = int(len((speech)-1)/chunk_stride+1)
+for i in range(total_chunk_num):
+    speech_chunk = speech[i*chunk_stride:(i+1)*chunk_stride]
+    is_final = i == total_chunk_num - 1
+    res = model.generate(input=speech_chunk, cache=cache, is_final=is_final, chunk_size=chunk_size)
+    if len(res[0]["value"]):
+        print(res)
+```
+
+#### 标点恢复
+```python
+from funasr import AutoModel
+
+model = AutoModel(model="ct-punc", model_revision="v2.0.4")
+
+res = model.generate(input="那今天的会就到这里吧 happy new year 明年见")
+print(res)
+```
+
+#### 时间戳预测
+```python
+from funasr import AutoModel
+
+model = AutoModel(model="fa-zh", model_revision="v2.0.4")
+
+wav_file = f"{model.model_path}/example/asr_example.wav"
+text_file = f"{model.model_path}/example/text.txt"
+res = model.generate(input=(wav_file, text_file), data_type=("sound", "text"))
+print(res)
+```
+
+更多详细用法��[示例](https://github.com/alibaba-damo-academy/FunASR/tree/main/examples/industrial_data_pretraining)）
+
+
+## 微调
+
+详细用法（[示例](https://github.com/alibaba-damo-academy/FunASR/tree/main/examples/industrial_data_pretraining)）
+
+
+
+
+
+## 使用方式以及适用范围
+
+运行范围
+- 支持Linux-x86_64、Mac和Windows运行。
+
+使用方式
+- 直接推理：可以直接对长语音数据进行计算，有效语音片段的起止时间点信息（单位：ms）。
+
+## 相关论文以及引用信息
+
+```BibTeX
+@inproceedings{zhang2018deep,
+  title={Deep-FSMN for large vocabulary continuous speech recognition},
+  author={Zhang, Shiliang and Lei, Ming and Yan, Zhijie and Dai, Lirong},
+  booktitle={2018 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
+  pages={5869--5873},
+  year={2018},
+  organization={IEEE}
+}
+```

FunASR/speech_fsmn_vad_zh-cn-16k-common-pytorch/am.mvn

@@ -0,0 +1,8 @@
+<Nnet>
+<Splice> 400 400
+[ 0 ]
+<AddShift> 400 400
+<LearnRateCoef> 0 [ -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 ]
+<Rescale> 400 400
+<LearnRateCoef> 0 [ 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 ]
+</Nnet>

FunASR/speech_fsmn_vad_zh-cn-16k-common-pytorch/config.yaml

@@ -0,0 +1,56 @@
+frontend: WavFrontendOnline
+frontend_conf:
+    fs: 16000
+    window: hamming
+    n_mels: 80
+    frame_length: 25
+    frame_shift: 10
+    dither: 0.0
+    lfr_m: 5
+    lfr_n: 1
+
+model: FsmnVADStreaming
+model_conf:
+    sample_rate: 16000
+    detect_mode: 1 
+    snr_mode: 0
+    max_end_silence_time: 800
+    max_start_silence_time: 3000
+    do_start_point_detection: True
+    do_end_point_detection: True
+    window_size_ms: 200
+    sil_to_speech_time_thres: 150
+    speech_to_sil_time_thres: 150
+    speech_2_noise_ratio: 1.0
+    do_extend: 1
+    lookback_time_start_point: 200
+    lookahead_time_end_point: 100
+    max_single_segment_time: 60000
+    snr_thres: -100.0
+    noise_frame_num_used_for_snr: 100
+    decibel_thres: -100.0
+    speech_noise_thres: 0.6
+    fe_prior_thres: 0.0001
+    silence_pdf_num: 1
+    sil_pdf_ids: [0]
+    speech_noise_thresh_low: -0.1
+    speech_noise_thresh_high: 0.3
+    output_frame_probs: False
+    frame_in_ms: 10
+    frame_length_ms: 25
+    
+encoder: FSMN
+encoder_conf:
+    input_dim: 400
+    input_affine_dim: 140
+    fsmn_layers: 4
+    linear_dim: 250
+    proj_dim: 128
+    lorder: 20
+    rorder: 0
+    lstride: 1
+    rstride: 0
+    output_affine_dim: 140
+    output_dim: 248
+
+

FunASR/speech_fsmn_vad_zh-cn-16k-common-pytorch/configuration.json

@@ -0,0 +1,13 @@
+{
+  "framework": "pytorch",
+  "task" : "voice-activity-detection",
+  "pipeline": {"type":"funasr-pipeline"},
+  "model": {"type" : "funasr"},
+  "file_path_metas": {
+    "init_param":"model.pt", 
+    "config":"config.yaml",
+    "frontend_conf":{"cmvn_file": "am.mvn"}},
+  "model_name_in_hub": {
+    "ms":"iic/speech_fsmn_vad_zh-cn-16k-common-pytorch", 
+    "hf":""}
+}

FunASR/speech_fsmn_vad_zh-cn-16k-common-pytorch/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3be75be477f0780277f3bae0fe489f48718f585f3a6e45d7dd1fbb1a4255fc5
+size 1721366

FunASR/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/README.md

@@ -0,0 +1,357 @@
+---
+tasks:
+- auto-speech-recognition
+domain:
+- audio
+model-type:
+- Non-autoregressive
+frameworks:
+- pytorch
+backbone:
+- transformer/conformer
+metrics:
+- CER
+license: Apache License 2.0
+language: 
+- cn
+tags:
+- FunASR
+- Paraformer
+- Alibaba
+- ICASSP2024
+- Hotword
+datasets:
+  train:
+  - 50,000 hour industrial Mandarin task
+  test:
+  - AISHELL-1-hotword dev/test
+indexing:
+   results:
+   - task:
+       name: Automatic Speech Recognition
+     dataset:
+       name: 50,000 hour industrial Mandarin task
+       type: audio    # optional
+       args: 16k sampling rate, 8404 characters  # optional
+     metrics:
+       - type: CER
+         value: 8.53%  # float
+         description: greedy search, withou lm, avg.
+         args: default
+       - type: RTF
+         value: 0.0251  # float
+         description: GPU inference on V100
+         args: batch_size=1
+widgets:
+  - task: auto-speech-recognition
+    inputs:
+      - type: audio
+        name: input
+        title: 音频
+    parameters:
+      - name: hotword
+        title: 热词
+        type: string
+    examples:
+      - name: 1
+        title: 示例1
+        inputs:
+          - name: input
+            data: git://example/asr_example.wav
+        parameters:
+          - name: hotword
+            value: 魔搭
+    model_revision: v2.0.4
+    inferencespec:
+      cpu: 8 #CPU数量
+      memory: 4096
+---
+
+# Paraformer-large模型介绍
+
+## Highlights
+Paraformer-large热词版模型支持热词定制功能：实现热词定制化功能，基于提供的热词列表进行激励增强，提升热词的召回率和准确率。
+
+
+## <strong>[FunASR开源项目介绍](https://github.com/alibaba-damo-academy/FunASR)</strong>
+<strong>[FunASR](https://github.com/alibaba-damo-academy/FunASR)</strong>希望在语音识别的学术研究和工业应用之间架起一座桥梁。通过发布工业级语音识别模型的训练和微调，研究人员和开发人员可以更方便地进行语音识别模型的研究和生产，并推动语音识别生态的发展。让语音识别更有趣！
+
+[**github仓库**](https://github.com/alibaba-damo-academy/FunASR) 
+|  [**最新动态**](https://github.com/alibaba-damo-academy/FunASR#whats-new) 
+| [**环境安装**](https://github.com/alibaba-damo-academy/FunASR#installation)
+| [**服务部署**](https://www.funasr.com)
+| [**模型库**](https://github.com/alibaba-damo-academy/FunASR/tree/main/model_zoo)
+| [**联系我们**](https://github.com/alibaba-damo-academy/FunASR#contact)
+
+
+## 模型原理介绍
+
+SeACoParaformer是阿里巴巴语音实验室提出的新一代热词定制化非自回归语音识别模型。相比于上一代基于CLAS的热词定制化方案，SeACoParaformer解耦了热词模块与ASR模型，通过后验概率融合的方式进行热词激励，使激励过程可见可控，并且热词召回率显著提升。
+
+<p align="center">
+<img src="fig/seaco.png" alt="SeACoParaformer模型结构"  width="380" />
+
+
+SeACoParaformer的模型结构与训练流程如上图所示，通过引入bias encoder进行热词embedding提取，bias decoder进行注意力建模，SeACoParaformer能够捕捉到Predictor输出和Decoder输出的信息与热词的相关性，并且预测与ASR结果同步的热词输出。通过后验概率的融合，实现热词激励。与ContextualParaformer相比，SeACoParaformer有明显的效果提升，如下图所示：
+
+<p align="center">
+<img src="fig/res.png" alt="SeACoParaformer模型结构"  width="700" />
+
+更详细的细节见：
+- 论文： [SeACo-Paraformer: A Non-Autoregressive ASR System with Flexible and Effective Hotword Customization Ability](https://arxiv.org/abs/2308.03266)
+
+## 复现论文中的结果
+```python
+from funasr import AutoModel
+
+model = AutoModel(model="iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
+                  model_revision="v2.0.4",
+                  # vad_model="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch",
+                  # vad_model_revision="v2.0.4",
+                  # punc_model="damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
+                  # punc_model_revision="v2.0.4",
+                  # spk_model="damo/speech_campplus_sv_zh-cn_16k-common",
+                  # spk_model_revision="v2.0.2",
+                  device="cuda:0"
+                  )
+
+res = model.generate(input="YOUR_PATH/aishell1_hotword_dev.scp",
+                     hotword='./data/dev/hotword.txt',
+                     batch_size_s=300,
+                    )
+fout1 = open("dev.output", 'w')
+for resi in res:
+    fout1.write("{}\t{}\n".format(resi['key'], resi['text']))
+
+res = model.generate(input="YOUR_PATH/aishell1_hotword_test.scp",
+                     hotword='./data/test/hotword.txt',
+                     batch_size_s=300,
+                    )
+fout2 = open("test.output", 'w')
+for resi in res:
+    fout2.write("{}\t{}\n".format(resi['key'], resi['text']))
+```
+
+## 基于ModelScope进行推理
+
+- 推理支��音频格式如下：
+  - wav文件路径，例如：data/test/audios/asr_example.wav
+  - pcm文件路径，例如：data/test/audios/asr_example.pcm
+  - wav文件url，例如：https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav
+  - wav二进制数据，格式bytes，例如：用户直接从文件里读出bytes数据或者是麦克风录出bytes数据。
+  - 已解析的audio音频，例如：audio, rate = soundfile.read("asr_example_zh.wav")，类型为numpy.ndarray或者torch.Tensor。
+  - wav.scp文件，需符合如下要求：
+
+```sh
+cat wav.scp
+asr_example1  data/test/audios/asr_example1.wav
+asr_example2  data/test/audios/asr_example2.wav
+...
+```
+
+- 若输入格式wav文件url，api调用方式可参考如下范例：
+
+```python
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+inference_pipeline = pipeline(
+    task=Tasks.auto_speech_recognition,
+    model='iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch', model_revision="v2.0.4")
+
+rec_result = inference_pipeline('https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav', hotword='达摩院 魔搭')
+print(rec_result)
+```
+
+- 输入音频为pcm格式，调用api时需要传入音频采样率参数audio_fs，例如：
+
+```python
+rec_result = inference_pipeline('https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.pcm', fs=16000, hotword='达摩院 魔搭')
+```
+
+- 输入音频为wav格式，api调用方式可参考如下范例:
+
+```python
+rec_result = inference_pipeline('asr_example_zh.wav', hotword='达摩院 魔搭')
+```
+
+- 若输入格式为文件wav.scp(注：文件名需要以.scp结尾)，可添加 output_dir 参数将识别结果写入文件中，api调用方式可参考如下范例:
+
+```python
+inference_pipeline("wav.scp", output_dir='./output_dir', hotword='达摩院 魔搭')
+```
+识别结果输出路径结构如下：
+
+```sh
+tree output_dir/
+output_dir/
+└── 1best_recog
+    ├── score
+    └── text
+
+1 directory, 3 files
+```
+
+score：识别路径得分
+
+text：语音识别结果文件
+
+
+- 若输入音频为已解析的audio音频，api调用方式可参考如下范例：
+
+```python
+import soundfile
+
+waveform, sample_rate = soundfile.read("asr_example_zh.wav")
+rec_result = inference_pipeline(waveform, hotword='达摩院 魔搭')
+```
+
+- ASR、VAD、PUNC模型自由组合
+
+可根据使用需求对VAD和PUNC标点模型进行自由组合，使用方式如下：
+```python
+inference_pipeline = pipeline(
+    task=Tasks.auto_speech_recognition,
+    model='iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch', model_revision="v2.0.4",
+    vad_model='iic/speech_fsmn_vad_zh-cn-16k-common-pytorch', vad_model_revision="v2.0.4",
+    punc_model='iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch', punc_model_revision="v2.0.3",
+    # spk_model="iic/speech_campplus_sv_zh-cn_16k-common",
+    # spk_model_revision="v2.0.2",
+)
+```
+若不使用PUNC模型，可配置punc_model=None，或不传入punc_model参数，如需加入LM模型，可增加配置lm_model='iic/speech_transformer_lm_zh-cn-common-vocab8404-pytorch'，并设置lm_weight和beam_size参数。
+
+## 基于FunASR进行推理
+
+下面为快速上手教程，测试音频（[中文](https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.wav)，[英文](https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_en.wav)）
+
+### 可执行命令行
+在命令行终端执行：
+
+```shell
+funasr +model=paraformer-zh +vad_model="fsmn-vad" +punc_model="ct-punc" +input=vad_example.wav
+```
+
+注：支持单条音频文件识别，也支持文件列表，列表为kaldi风格wav.scp：`wav_id   wav_path`
+
+### python示例
+#### 非实时语音识别
+```python
+from funasr import AutoModel
+# paraformer-zh is a multi-functional asr model
+# use vad, punc, spk or not as you need
+model = AutoModel(model="paraformer-zh", model_revision="v2.0.4",
+                  vad_model="fsmn-vad", vad_model_revision="v2.0.4",
+                  punc_model="ct-punc-c", punc_model_revision="v2.0.4",
+                  # spk_model="cam++", spk_model_revision="v2.0.2",
+                  )
+res = model.generate(input=f"{model.model_path}/example/asr_example.wav", 
+            batch_size_s=300, 
+            hotword='魔搭')
+print(res)
+```
+注：`model_hub`：表示模型仓库，`ms`为选择modelscope下载，`hf`为选择huggingface下载。
+
+#### 实时语音识别
+
+```python
+from funasr import AutoModel
+
+chunk_size = [0, 10, 5] #[0, 10, 5] 600ms, [0, 8, 4] 480ms
+encoder_chunk_look_back = 4 #number of chunks to lookback for encoder self-attention
+decoder_chunk_look_back = 1 #number of encoder chunks to lookback for decoder cross-attention
+
+model = AutoModel(model="paraformer-zh-streaming", model_revision="v2.0.4")
+
+import soundfile
+import os
+
+wav_file = os.path.join(model.model_path, "example/asr_example.wav")
+speech, sample_rate = soundfile.read(wav_file)
+chunk_stride = chunk_size[1] * 960 # 600ms
+
+cache = {}
+total_chunk_num = int(len((speech)-1)/chunk_stride+1)
+for i in range(total_chunk_num):
+    speech_chunk = speech[i*chunk_stride:(i+1)*chunk_stride]
+    is_final = i == total_chunk_num - 1
+    res = model.generate(input=speech_chunk, cache=cache, is_final=is_final, chunk_size=chunk_size, encoder_chunk_look_back=encoder_chunk_look_back, decoder_chunk_look_back=decoder_chunk_look_back)
+    print(res)
+```
+
+注：`chunk_size`为流式延时配置，`[0,10,5]`表示上屏实时出字粒度为`10*60=600ms`，未来信息为`5*60=300ms`。每次推理输入为`600ms`（采样点数为`16000*0.6=960`），输出为对应文字，最后一个语音片段输入需要设置`is_final=True`来强制输出最后一个字。
+
+#### 语音端点检测（非实时）
+```python
+from funasr import AutoModel
+
+model = AutoModel(model="fsmn-vad", model_revision="v2.0.4")
+
+wav_file = f"{model.model_path}/example/asr_example.wav"
+res = model.generate(input=wav_file)
+print(res)
+```
+
+#### 语音端点检测（实时）
+```python
+from funasr import AutoModel
+
+chunk_size = 200 # ms
+model = AutoModel(model="fsmn-vad", model_revision="v2.0.4")
+
+import soundfile
+
+wav_file = f"{model.model_path}/example/vad_example.wav"
+speech, sample_rate = soundfile.read(wav_file)
+chunk_stride = int(chunk_size * sample_rate / 1000)
+
+cache = {}
+total_chunk_num = int(len((speech)-1)/chunk_stride+1)
+for i in range(total_chunk_num):
+    speech_chunk = speech[i*chunk_stride:(i+1)*chunk_stride]
+    is_final = i == total_chunk_num - 1
+    res = model.generate(input=speech_chunk, cache=cache, is_final=is_final, chunk_size=chunk_size)
+    if len(res[0]["value"]):
+        print(res)
+```
+
+#### 标点恢复
+```python
+from funasr import AutoModel
+
+model = AutoModel(model="ct-punc", model_revision="v2.0.4")
+
+res = model.generate(input="那今天的会就到这里吧 happy new year 明年见")
+print(res)
+```
+
+#### 时间戳预测
+```python
+from funasr import AutoModel
+
+model = AutoModel(model="fa-zh", model_revision="v2.0.4")
+
+wav_file = f"{model.model_path}/example/asr_example.wav"
+text_file = f"{model.model_path}/example/text.txt"
+res = model.generate(input=(wav_file, text_file), data_type=("sound", "text"))
+print(res)
+```
+
+更多详细用法（[示例](https://github.com/alibaba-damo-academy/FunASR/tree/main/examples/industrial_data_pretraining)）
+
+
+## 微调
+
+详细用法（[示例](https://github.com/alibaba-damo-academy/FunASR/tree/main/examples/industrial_data_pretraining)）
+
+
+## 相关论文以及引用信息
+
+```BibTeX
+@article{shi2023seaco,
+  title={SeACo-Paraformer: A Non-Autoregressive ASR System with Flexible and Effective Hotword Customization Ability},
+  author={Shi, Xian and Yang, Yexin and Li, Zerui and Zhang, Shiliang},
+  journal={arXiv preprint arXiv:2308.03266 (accepted by ICASSP2024)},
+  year={2023}
+}
+```

FunASR/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/am.mvn

@@ -0,0 +1,8 @@
+<Nnet> 
+<Splice> 560 560
+[ 0 ]
+<AddShift> 560 560 
+<LearnRateCoef> 0 [ -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 ]
+<Rescale> 560 560
+<LearnRateCoef> 0 [ 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 ]
+</Nnet> 

FunASR/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/config.yaml

@@ -0,0 +1,159 @@
+# This is an example that demonstrates how to configure a model file.
+# You can modify the configuration according to your own requirements.
+
+# to print the register_table:
+# from funasr.utils.register import registry_tables
+# registry_tables.print()
+
+# network architecture
+model: SeacoParaformer
+model_conf:
+    ctc_weight: 0.0
+    lsm_weight: 0.1
+    length_normalized_loss: true
+    predictor_weight: 1.0
+    predictor_bias: 1
+    sampling_ratio: 0.75
+    inner_dim: 512
+    bias_encoder_type: lstm
+    bias_encoder_bid: false
+    seaco_lsm_weight: 0.1
+    seaco_length_normal: true
+    train_decoder: true
+    NO_BIAS: 8377
+
+# encoder
+encoder: SANMEncoder
+encoder_conf:
+    output_size: 512
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 50
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: pe
+    pos_enc_class: SinusoidalPositionEncoder
+    normalize_before: true
+    kernel_size: 11
+    sanm_shfit: 0
+    selfattention_layer_type: sanm
+
+# decoder
+decoder: ParaformerSANMDecoder
+decoder_conf:
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 16
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+    att_layer_num: 16
+    kernel_size: 11
+    sanm_shfit: 0
+
+# seaco decoder
+seaco_decoder: ParaformerSANMDecoder
+seaco_decoder_conf:
+    attention_heads: 4
+    linear_units: 1024
+    num_blocks: 4
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+    kernel_size: 21
+    sanm_shfit: 0
+    use_output_layer: false
+    wo_input_layer: true
+
+predictor: CifPredictorV3
+predictor_conf:
+    idim: 512
+    threshold: 1.0
+    l_order: 1
+    r_order: 1
+    tail_threshold: 0.45
+    smooth_factor2: 0.25
+    noise_threshold2: 0.01
+    upsample_times: 3
+    use_cif1_cnn: false
+    upsample_type: cnn_blstm
+
+# frontend related
+frontend: WavFrontend
+frontend_conf:
+    fs: 16000
+    window: hamming
+    n_mels: 80
+    frame_length: 25
+    frame_shift: 10
+    lfr_m: 7
+    lfr_n: 6
+    dither: 0.0
+
+specaug: SpecAugLFR
+specaug_conf:
+    apply_time_warp: false
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    lfr_rate: 6
+    num_freq_mask: 1
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 12
+    num_time_mask: 1
+
+train_conf:
+  accum_grad: 1
+  grad_clip: 5
+  max_epoch: 150
+  val_scheduler_criterion:
+      - valid
+      - acc
+  best_model_criterion:
+  -   - valid
+      - acc
+      - max
+  keep_nbest_models: 10
+  log_interval: 50
+
+optim: adam
+optim_conf:
+   lr: 0.0005
+scheduler: warmuplr
+scheduler_conf:
+   warmup_steps: 30000
+
+dataset: AudioDatasetHotword
+dataset_conf:
+    seaco_id: 8377
+    index_ds: IndexDSJsonl
+    batch_sampler: DynamicBatchLocalShuffleSampler
+    batch_type: example # example or length
+    batch_size: 1 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
+    max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length,
+    buffer_size: 500
+    shuffle: True
+    num_workers: 0
+
+tokenizer: CharTokenizer
+tokenizer_conf:
+  unk_symbol: <unk>
+  split_with_space: true
+
+
+ctc_conf:
+    dropout_rate: 0.0
+    ctc_type: builtin
+    reduce: true
+    ignore_nan_grad: true
+    
+normalize: null
+unused_parameters: true

FunASR/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/configuration.json

@@ -0,0 +1,14 @@
+{
+  "framework": "pytorch",
+  "task" : "auto-speech-recognition",
+  "model": {"type" : "funasr"},
+  "pipeline": {"type":"funasr-pipeline"},
+  "model_name_in_hub": {
+    "ms":"iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch", 
+    "hf":""},
+  "file_path_metas": {
+    "init_param":"model.pt", 
+    "config":"config.yaml",
+    "tokenizer_conf": {"token_list": "tokens.json", "seg_dict_file": "seg_dict"},
+    "frontend_conf":{"cmvn_file": "am.mvn"}}
+}

FunASR/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/example/hotword.txt

@@ -0,0 +1 @@
+魔搭

FunASR/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d491689244ec5dfbf9170ef3827c358aa10f1f20e42a7c59e15e688647946d1
+size 989763045

GPT_SoVITS/AR/data/bucket_sampler.py

@@ -0,0 +1,162 @@
+# modified from https://github.com/feng-yufei/shared_debugging_code/blob/main/bucketsampler.py
+import itertools
+import math
+import random
+from random import shuffle
+from typing import Iterator
+from typing import Optional
+from typing import TypeVar
+
+import torch
+import torch.distributed as dist
+from torch.utils.data import Dataset
+from torch.utils.data import Sampler
+
+__all__ = [
+    "DistributedBucketSampler",
+]
+
+T_co = TypeVar("T_co", covariant=True)
+
+
+class DistributedBucketSampler(Sampler[T_co]):
+    r"""
+    sort the dataset wrt. input length
+    divide samples into buckets
+    sort within buckets
+    divide buckets into batches
+    sort batches
+    """
+
+    def __init__(
+        self,
+        dataset: Dataset,
+        num_replicas: Optional[int] = None,
+        rank: Optional[int] = None,
+        shuffle: bool = True,
+        seed: int = 0,
+        drop_last: bool = False,
+        batch_size: int = 32,
+    ) -> None:
+        if num_replicas is None:
+            if not dist.is_available():
+                raise RuntimeError("Requires distributed package to be available")
+            num_replicas = dist.get_world_size() if torch.cuda.is_available() else 1
+        if rank is None:
+            if not dist.is_available():
+                raise RuntimeError("Requires distributed package to be available")
+            rank = dist.get_rank() if torch.cuda.is_available() else 0
+            if torch.cuda.is_available():
+                torch.cuda.set_device(rank)
+        if rank >= num_replicas or rank < 0:
+            raise ValueError(
+                "Invalid rank {}, rank should be in the interval"
+                " [0, {}]".format(rank, num_replicas - 1)
+            )
+        self.dataset = dataset
+        self.num_replicas = num_replicas
+        self.rank = rank
+        self.epoch = 0
+        self.drop_last = drop_last
+        # If the dataset length is evenly divisible by # of replicas, then there
+        # is no need to drop any data, since the dataset will be split equally.
+        if (
+            self.drop_last and len(self.dataset) % self.num_replicas != 0
+        ):  # type: ignore[arg-type]
+            # Split to nearest available length that is evenly divisible.
+            # This is to ensure each rank receives the same amount of data when
+            # using this Sampler.
+            self.num_samples = math.ceil(
+                (len(self.dataset) - self.num_replicas)
+                / self.num_replicas  # type: ignore[arg-type]
+            )
+        else:
+            self.num_samples = math.ceil(
+                len(self.dataset) / self.num_replicas
+            )  # type: ignore[arg-type]
+        self.total_size = self.num_samples * self.num_replicas
+        self.shuffle = shuffle
+        self.seed = seed
+        self.batch_size = batch_size
+        self.id_with_length = self._get_sample_lengths()
+        self.id_buckets = self.make_buckets(bucket_width=2.0)
+
+    def _get_sample_lengths(self):
+        id_with_lengths = []
+        for i in range(len(self.dataset)):
+            id_with_lengths.append((i, self.dataset.get_sample_length(i)))
+        id_with_lengths.sort(key=lambda x: x[1])
+        return id_with_lengths
+
+    def make_buckets(self, bucket_width: float = 2.0):
+        buckets = []
+        cur = []
+        max_sec = bucket_width
+        for id, sec in self.id_with_length:
+            if sec < max_sec:
+                cur.append(id)
+            else:
+                buckets.append(cur)
+                cur = [id]
+                max_sec += bucket_width
+        if len(cur) > 0:
+            buckets.append(cur)
+        return buckets
+
+    def __iter__(self) -> Iterator[T_co]:
+        if self.shuffle:
+            # deterministically shuffle based on epoch and seed
+            g = torch.Generator()
+            g.manual_seed(self.seed + self.epoch)
+            random.seed(self.epoch + self.seed)
+            shuffled_bucket = []
+            for buc in self.id_buckets:
+                buc_copy = buc.copy()
+                shuffle(buc_copy)
+                shuffled_bucket.append(buc_copy)
+            grouped_batch_size = self.batch_size * self.num_replicas
+            shuffled_bucket = list(itertools.chain(*shuffled_bucket))
+            n_batch = int(math.ceil(len(shuffled_bucket) / grouped_batch_size))
+            batches = [
+                shuffled_bucket[b * grouped_batch_size : (b + 1) * grouped_batch_size]
+                for b in range(n_batch)
+            ]
+            shuffle(batches)
+            indices = list(itertools.chain(*batches))
+        else:
+            # type: ignore[arg-type]
+            indices = list(range(len(self.dataset)))
+
+        if not self.drop_last:
+            # add extra samples to make it evenly divisible
+            padding_size = self.total_size - len(indices)
+            if padding_size <= len(indices):
+                indices += indices[:padding_size]
+            else:
+                indices += (indices * math.ceil(padding_size / len(indices)))[
+                    :padding_size
+                ]
+        else:
+            # remove tail of data to make it evenly divisible.
+            indices = indices[: self.total_size]
+        assert len(indices) == self.total_size
+
+        # subsample
+        indices = indices[self.rank : self.total_size : self.num_replicas]
+        assert len(indices) == self.num_samples
+
+        return iter(indices)
+
+    def __len__(self) -> int:
+        return self.num_samples
+
+    def set_epoch(self, epoch: int) -> None:
+        r"""
+        Sets the epoch for this sampler. When :attr:`shuffle=True`, this ensures all replicas
+        use a different random ordering for each epoch. Otherwise, the next iteration of this
+        sampler will yield the same ordering.
+
+        Args:
+            epoch (int): Epoch number.
+        """
+        self.epoch = epoch

GPT_SoVITS/AR/data/data_module.py

@@ -0,0 +1,74 @@
+# modified from https://github.com/feng-yufei/shared_debugging_code/blob/main/data_module.py
+from pytorch_lightning import LightningDataModule
+from AR.data.bucket_sampler import DistributedBucketSampler
+from AR.data.dataset import Text2SemanticDataset
+from torch.utils.data import DataLoader
+
+
+class Text2SemanticDataModule(LightningDataModule):
+    def __init__(
+        self,
+        config,
+        train_semantic_path,
+        train_phoneme_path,
+        dev_semantic_path=None,
+        dev_phoneme_path=None,
+    ):
+        super().__init__()
+        self.config = config
+        self.train_semantic_path = train_semantic_path
+        self.train_phoneme_path = train_phoneme_path
+        self.dev_semantic_path = dev_semantic_path
+        self.dev_phoneme_path = dev_phoneme_path
+        self.num_workers = self.config["data"]["num_workers"]
+
+    def prepare_data(self):
+        pass
+
+    def setup(self, stage=None, output_logs=False):
+        self._train_dataset = Text2SemanticDataset(
+            phoneme_path=self.train_phoneme_path,
+            semantic_path=self.train_semantic_path,
+            max_sec=self.config["data"]["max_sec"],
+            pad_val=self.config["data"]["pad_val"],
+        )
+        self._dev_dataset = self._train_dataset
+        # self._dev_dataset = Text2SemanticDataset(
+        #     phoneme_path=self.dev_phoneme_path,
+        #     semantic_path=self.dev_semantic_path,
+        #     max_sample=self.config['data']['max_eval_sample'],
+        #     max_sec=self.config['data']['max_sec'],
+        #     pad_val=self.config['data']['pad_val'])
+
+    def train_dataloader(self):
+        batch_size = max(min(self.config["train"]["batch_size"],len(self._train_dataset)//4),1)#防止不保存
+        sampler = DistributedBucketSampler(self._train_dataset, batch_size=batch_size)
+        return DataLoader(
+            self._train_dataset,
+            batch_size=batch_size,
+            sampler=sampler,
+            collate_fn=self._train_dataset.collate,
+            num_workers=self.num_workers,
+            persistent_workers=True,
+            prefetch_factor=16,
+        )
+
+    def val_dataloader(self):
+        return DataLoader(
+            self._dev_dataset,
+            batch_size=1,
+            shuffle=False,
+            collate_fn=self._train_dataset.collate,
+            num_workers=max(self.num_workers, 12),
+            persistent_workers=True,
+            prefetch_factor=16,
+        )
+
+    # 这个会使用到嘛？
+    def test_dataloader(self):
+        return DataLoader(
+            self._dev_dataset,
+            batch_size=1,
+            shuffle=False,
+            collate_fn=self._train_dataset.collate,
+        )

GPT_SoVITS/AR/data/dataset.py

@@ -0,0 +1,320 @@
+# modified from https://github.com/feng-yufei/shared_debugging_code/blob/main/t2s_dataset.py
+import pdb
+import sys
+
+# sys.path.append("/data/docker/liujing04/gpt-vits/mq-vits-s1bert_no_bert")
+import traceback, os
+from typing import Dict
+from typing import List
+
+import numpy as np
+import pandas as pd
+import torch, json
+from torch.utils.data import DataLoader
+from torch.utils.data import Dataset
+from transformers import AutoTokenizer
+
+from text import cleaned_text_to_sequence
+
+# from config import exp_dir
+
+
+def batch_sequences(sequences: List[np.array], axis: int = 0, pad_value: int = 0):
+    seq = sequences[0]
+    ndim = seq.ndim
+    if axis < 0:
+        axis += ndim
+    dtype = seq.dtype
+    pad_value = dtype.type(pad_value)
+    seq_lengths = [seq.shape[axis] for seq in sequences]
+    max_length = np.max(seq_lengths)
+
+    padded_sequences = []
+    for seq, length in zip(sequences, seq_lengths):
+        padding = (
+            [(0, 0)] * axis + [(0, max_length - length)] + [(0, 0)] * (ndim - axis - 1)
+        )
+        padded_seq = np.pad(seq, padding, mode="constant", constant_values=pad_value)
+        padded_sequences.append(padded_seq)
+    batch = np.stack(padded_sequences)
+    return batch
+
+
+class Text2SemanticDataset(Dataset):
+    """dataset class for text tokens to semantic model training."""
+
+    def __init__(
+        self,
+        phoneme_path: str,
+        semantic_path: str,
+        max_sample: int = None,
+        max_sec: int = 100,
+        pad_val: int = 1024,
+        # min value of phoneme/sec
+        min_ps_ratio: int = 3,
+        # max value of phoneme/sec
+        max_ps_ratio: int = 25,
+    ) -> None:
+        super().__init__()
+
+        self.semantic_data = pd.read_csv(
+            semantic_path, delimiter="\t", encoding="utf-8"
+        )
+        # get dict
+        self.path2 = phoneme_path  # "%s/2-name2text.txt"%exp_dir#phoneme_path
+        self.path3 = "%s/3-bert" % (
+            os.path.basename(phoneme_path)
+        )  # "%s/3-bert"%exp_dir#bert_dir
+        self.path6 = semantic_path  # "%s/6-name2semantic.tsv"%exp_dir#semantic_path
+        assert os.path.exists(self.path2)
+        assert os.path.exists(self.path6)
+        self.phoneme_data = {}
+        with open(self.path2, "r", encoding="utf8") as f:
+            lines = f.read().strip("\n").split("\n")
+
+        for line in lines:
+            tmp = line.split("\t")
+            if len(tmp) != 4:
+                continue
+            self.phoneme_data[tmp[0]] = [tmp[1], tmp[2], tmp[3]]
+
+        # self.phoneme_data = np.load(phoneme_path, allow_pickle=True).item()
+        # pad for semantic tokens
+        self.PAD: int = pad_val
+        # self.hz = 25
+        # with open("/data/docker/liujing04/gpt-vits/mq-vits-s1bert_no_bert/configs/s2.json", "r") as f:data = f.read()
+        # data=json.loads(data)["model"]["semantic_frame_rate"]#50hz
+        # self.hz=int(data[:-2])#
+        self.hz = int(os.environ.get("hz", "25hz")[:-2])
+
+        # max seconds of semantic token
+        self.max_sec = max_sec
+        self.min_ps_ratio = min_ps_ratio
+        self.max_ps_ratio = max_ps_ratio
+
+        if max_sample is not None:
+            self.semantic_data = self.semantic_data[:max_sample]
+
+        # {idx: (semantic, phoneme)}
+        # semantic list, phoneme list
+        self.semantic_phoneme = []
+        self.item_names = []
+
+        self.inited = False
+
+        if not self.inited:
+            # 调用初始化函数
+            self.init_batch()
+            self.inited = True
+            del self.semantic_data
+            del self.phoneme_data
+        # self.tokenizer = AutoTokenizer.from_pretrained("hfl/chinese-roberta-wwm-ext-large")
+        # self.tokenizer = AutoTokenizer.from_pretrained("/data/docker/liujing04/bert-vits2/Bert-VITS2-master20231106/bert/chinese-roberta-wwm-ext-large")
+
+    def init_batch(self):
+        semantic_data_len = len(self.semantic_data)
+        phoneme_data_len = len(self.phoneme_data.keys())
+        print("semantic_data_len:", semantic_data_len)
+        print("phoneme_data_len:", phoneme_data_len)
+        print(self.semantic_data)
+        idx = 0
+        num_not_in = 0
+        num_deleted_bigger = 0
+        num_deleted_ps = 0
+        for i in range(semantic_data_len):
+            # 先依次遍历
+            # get str
+            item_name = self.semantic_data.iloc[i,0]
+            # print(self.phoneme_data)
+            try:
+                phoneme, word2ph, text = self.phoneme_data[item_name]
+            except Exception:
+                traceback.print_exc()
+                # print(f"{item_name} not in self.phoneme_data !")
+                num_not_in += 1
+                continue
+
+            semantic_str = self.semantic_data.iloc[i,1]
+            # get token list
+            semantic_ids = [int(idx) for idx in semantic_str.split(" ")]
+            # (T), 是否需要变成 (1, T) -> 不需要，因为需要求 len
+            # 过滤掉太长的样本
+            if (
+                len(semantic_ids) > self.max_sec * self.hz
+            ):  #########1###根据token个数推测总时长过滤时长60s（config里）#40*25=1k
+                num_deleted_bigger += 1
+                continue
+            # (T, ), 这个速度不会很慢，所以可以在一开始就处理，无需在 __getitem__ 里面单个处理####
+            phoneme = phoneme.split(" ")
+
+            try:
+                phoneme_ids = cleaned_text_to_sequence(phoneme)
+            except:
+                traceback.print_exc()
+                # print(f"{item_name} not in self.phoneme_data !")
+                num_not_in += 1
+                continue
+            # if len(phoneme_ids) >400:###########2：改为恒定限制为semantic/2.5就行
+            if (
+                len(phoneme_ids) > self.max_sec * self.hz / 2.5
+            ):  ###########2：改为恒定限制为semantic/2.5就行
+                num_deleted_ps += 1
+                continue
+            # if len(semantic_ids) > 1000:###########3
+            #     num_deleted_bigger += 1
+            #     continue
+
+            ps_ratio = len(phoneme_ids) / (len(semantic_ids) / self.hz)
+
+            if (
+                ps_ratio > self.max_ps_ratio or ps_ratio < self.min_ps_ratio
+            ):  ##########4#3~25#每秒多少个phone
+                num_deleted_ps += 1
+                # print(item_name)
+                continue
+
+            self.semantic_phoneme.append((semantic_ids, phoneme_ids))
+            idx += 1
+            self.item_names.append(item_name)
+
+        min_num = 100  # 20直接不补#30补了也不存ckpt
+        leng = len(self.semantic_phoneme)
+        if leng < min_num:
+            tmp1 = self.semantic_phoneme
+            tmp2 = self.item_names
+            self.semantic_phoneme = []
+            self.item_names = []
+            for _ in range(max(2, int(min_num / leng))):
+                self.semantic_phoneme += tmp1
+                self.item_names += tmp2
+        if num_not_in > 0:
+            print(f"there are {num_not_in} semantic datas not in phoneme datas")
+        if num_deleted_bigger > 0:
+            print(
+                f"deleted {num_deleted_bigger} audios who's duration are bigger than {self.max_sec} seconds"
+            )
+        if num_deleted_ps > 0:
+            # 4702 for LibriTTS, LirbriTTS 是标注数据, 是否需要筛？=> 需要，有值为 100 的极端值
+            print(
+                f"deleted {num_deleted_ps} audios who's phoneme/sec are bigger than {self.max_ps_ratio} or smaller than {self.min_ps_ratio}"
+            )
+        """
+        there are 31 semantic datas not in phoneme datas
+        deleted 34 audios who's duration are bigger than 54 seconds
+        deleted 3190 audios who's phoneme/sec are bigger than 25 or smaller than 3
+        dataset.__len__(): 366463
+
+        """
+        # 345410 for LibriTTS
+        print("dataset.__len__():", self.__len__())
+
+    def __get_item_names__(self) -> List[str]:
+        return self.item_names
+
+    def __len__(self) -> int:
+        return len(self.semantic_phoneme)
+
+    def __getitem__(self, idx: int) -> Dict:
+        semantic_ids, phoneme_ids = self.semantic_phoneme[idx]
+        item_name = self.item_names[idx]
+        phoneme_ids_len = len(phoneme_ids)
+        # semantic tokens target
+        semantic_ids_len = len(semantic_ids)
+
+        flag = 0
+        path_bert = "%s/%s.pt" % (self.path3, item_name)
+        if os.path.exists(path_bert) == True:
+            bert_feature = torch.load(path_bert, map_location="cpu")
+        else:
+            flag = 1
+        if flag == 1:
+            # bert_feature=torch.zeros_like(phoneme_ids,dtype=torch.float32)
+            bert_feature = None
+        else:
+            assert bert_feature.shape[-1] == len(phoneme_ids)
+        return {
+            "idx": idx,
+            "phoneme_ids": phoneme_ids,
+            "phoneme_ids_len": phoneme_ids_len,
+            "semantic_ids": semantic_ids,
+            "semantic_ids_len": semantic_ids_len,
+            "bert_feature": bert_feature,
+        }
+
+    def get_sample_length(self, idx: int):
+        semantic_ids = self.semantic_phoneme[idx][0]
+        sec = 1.0 * len(semantic_ids) / self.hz
+        return sec
+
+    def collate(self, examples: List[Dict]) -> Dict:
+        sample_index: List[int] = []
+        phoneme_ids: List[torch.Tensor] = []
+        phoneme_ids_lens: List[int] = []
+        semantic_ids: List[torch.Tensor] = []
+        semantic_ids_lens: List[int] = []
+        # return
+
+        for item in examples:
+            sample_index.append(item["idx"])
+            phoneme_ids.append(np.array(item["phoneme_ids"], dtype=np.int64))
+            semantic_ids.append(np.array(item["semantic_ids"], dtype=np.int64))
+            phoneme_ids_lens.append(item["phoneme_ids_len"])
+            semantic_ids_lens.append(item["semantic_ids_len"])
+
+        # pad 0
+        phoneme_ids = batch_sequences(phoneme_ids)
+        semantic_ids = batch_sequences(semantic_ids, pad_value=self.PAD)
+
+        # # convert each batch to torch.tensor
+        phoneme_ids = torch.tensor(phoneme_ids)
+        semantic_ids = torch.tensor(semantic_ids)
+        phoneme_ids_lens = torch.tensor(phoneme_ids_lens)
+        semantic_ids_lens = torch.tensor(semantic_ids_lens)
+        bert_padded = torch.FloatTensor(len(examples), 1024, max(phoneme_ids_lens))
+        bert_padded.zero_()
+
+        for idx, item in enumerate(examples):
+            bert = item["bert_feature"]
+            if bert != None:
+                bert_padded[idx, :, : bert.shape[-1]] = bert
+
+        return {
+            # List[int]
+            "ids": sample_index,
+            # torch.Tensor (B, max_phoneme_length)
+            "phoneme_ids": phoneme_ids,
+            # torch.Tensor (B)
+            "phoneme_ids_len": phoneme_ids_lens,
+            # torch.Tensor (B, max_semantic_ids_length)
+            "semantic_ids": semantic_ids,
+            # torch.Tensor (B)
+            "semantic_ids_len": semantic_ids_lens,
+            # torch.Tensor (B, 1024, max_phoneme_length)
+            "bert_feature": bert_padded,
+        }
+
+
+if __name__ == "__main__":
+    root_dir = "/data/docker/liujing04/gpt-vits/prepare/dump_mix/"
+    dataset = Text2SemanticDataset(
+        phoneme_path=root_dir + "phoneme_train.npy",
+        semantic_path=root_dir + "semantic_train.tsv",
+    )
+
+    batch_size = 12
+    dataloader = DataLoader(
+        dataset, batch_size=batch_size, collate_fn=dataset.collate, shuffle=False
+    )
+    for i, batch in enumerate(dataloader):
+        if i % 1000 == 0:
+            print(i)
+        # if i == 0:
+        #     print('batch["ids"]:', batch["ids"])
+        # print('batch["phoneme_ids"]:', batch["phoneme_ids"],
+        #       batch["phoneme_ids"].shape)
+        # print('batch["phoneme_ids_len"]:', batch["phoneme_ids_len"],
+        #       batch["phoneme_ids_len"].shape)
+        # print('batch["semantic_ids"]:', batch["semantic_ids"],
+        #       batch["semantic_ids"].shape)
+        # print('batch["semantic_ids_len"]:', batch["semantic_ids_len"],
+        #       batch["semantic_ids_len"].shape)

GPT_SoVITS/AR/models/t2s_lightning_module.py

@@ -0,0 +1,140 @@
+# modified from https://github.com/feng-yufei/shared_debugging_code/blob/main/model/t2s_lightning_module.py
+import os, sys
+
+now_dir = os.getcwd()
+sys.path.append(now_dir)
+from typing import Dict
+
+import torch
+from pytorch_lightning import LightningModule
+from AR.models.t2s_model import Text2SemanticDecoder
+from AR.modules.lr_schedulers import WarmupCosineLRSchedule
+from AR.modules.optim import ScaledAdam
+
+
+class Text2SemanticLightningModule(LightningModule):
+    def __init__(self, config, output_dir, is_train=True):
+        super().__init__()
+        self.config = config
+        self.top_k = 3
+        self.model = Text2SemanticDecoder(config=config, top_k=self.top_k)
+        pretrained_s1 = config.get("pretrained_s1")
+        if pretrained_s1 and is_train:
+            # print(self.load_state_dict(torch.load(pretrained_s1,map_location="cpu")["state_dict"]))
+            print(
+                self.load_state_dict(
+                    torch.load(pretrained_s1, map_location="cpu")["weight"]
+                )
+            )
+        if is_train:
+            self.automatic_optimization = False
+            self.save_hyperparameters()
+            self.eval_dir = output_dir / "eval"
+            self.eval_dir.mkdir(parents=True, exist_ok=True)
+
+    def training_step(self, batch: Dict, batch_idx: int):
+        opt = self.optimizers()
+        scheduler = self.lr_schedulers()
+        loss, acc = self.model.forward(
+            batch["phoneme_ids"],
+            batch["phoneme_ids_len"],
+            batch["semantic_ids"],
+            batch["semantic_ids_len"],
+            batch["bert_feature"],
+        )
+        self.manual_backward(loss)
+        if batch_idx > 0 and batch_idx % 4 == 0:
+            opt.step()
+            opt.zero_grad()
+            scheduler.step()
+
+        self.log(
+            "total_loss",
+            loss,
+            on_step=True,
+            on_epoch=True,
+            prog_bar=True,
+            sync_dist=True,
+        )
+        self.log(
+            "lr",
+            scheduler.get_last_lr()[0],
+            on_epoch=True,
+            prog_bar=True,
+            sync_dist=True,
+        )
+        self.log(
+            f"top_{self.top_k}_acc",
+            acc,
+            on_step=True,
+            on_epoch=True,
+            prog_bar=True,
+            sync_dist=True,
+        )
+
+    def validation_step(self, batch: Dict, batch_idx: int):
+        return
+
+    # # get loss
+    # loss, acc = self.model.forward(
+    #     batch['phoneme_ids'], batch['phoneme_ids_len'],
+    #     batch['semantic_ids'], batch['semantic_ids_len'],
+    #     batch['bert_feature']
+    # )
+    #
+    # self.log(
+    #     "val_total_loss",
+    #     loss,
+    #     on_step=True,
+    #     on_epoch=True,
+    #     prog_bar=True,
+    #     sync_dist=True)
+    # self.log(
+    #     f"val_top_{self.top_k}_acc",
+    #     acc,
+    #     on_step=True,
+    #     on_epoch=True,
+    #     prog_bar=True,
+    #     sync_dist=True)
+    #
+    # # get infer output
+    # semantic_len = batch['semantic_ids'].size(1)
+    # prompt_len = min(int(semantic_len * 0.5), 150)
+    # prompt = batch['semantic_ids'][:, :prompt_len]
+    # pred_semantic = self.model.infer(batch['phoneme_ids'],
+    #                                  batch['phoneme_ids_len'], prompt,
+    #                                  batch['bert_feature']
+    #                                  )
+    # save_name = f'semantic_toks_{batch_idx}.pt'
+    # save_path = os.path.join(self.eval_dir, save_name)
+    # torch.save(pred_semantic.detach().cpu(), save_path)
+
+    def configure_optimizers(self):
+        model_parameters = self.model.parameters()
+        parameters_names = []
+        parameters_names.append(
+            [name_param_pair[0] for name_param_pair in self.model.named_parameters()]
+        )
+        lm_opt = ScaledAdam(
+            model_parameters,
+            lr=0.01,
+            betas=(0.9, 0.95),
+            clipping_scale=2.0,
+            parameters_names=parameters_names,
+            show_dominant_parameters=False,
+            clipping_update_period=1000,
+        )
+
+        return {
+            "optimizer": lm_opt,
+            "lr_scheduler": {
+                "scheduler": WarmupCosineLRSchedule(
+                    lm_opt,
+                    init_lr=self.config["optimizer"]["lr_init"],
+                    peak_lr=self.config["optimizer"]["lr"],
+                    end_lr=self.config["optimizer"]["lr_end"],
+                    warmup_steps=self.config["optimizer"]["warmup_steps"],
+                    total_steps=self.config["optimizer"]["decay_steps"],
+                )
+            },
+        }

GPT_SoVITS/AR/models/t2s_lightning_module_onnx.py

@@ -0,0 +1,106 @@
+# modified from https://github.com/feng-yufei/shared_debugging_code/blob/main/model/t2s_lightning_module.py
+import os, sys
+
+now_dir = os.getcwd()
+sys.path.append(now_dir)
+from typing import Dict
+
+import torch
+from pytorch_lightning import LightningModule
+from AR.models.t2s_model_onnx import Text2SemanticDecoder
+from AR.modules.lr_schedulers import WarmupCosineLRSchedule
+from AR.modules.optim import ScaledAdam
+
+
+class Text2SemanticLightningModule(LightningModule):
+    def __init__(self, config, output_dir, is_train=True):
+        super().__init__()
+        self.config = config
+        self.top_k = 3
+        self.model = Text2SemanticDecoder(config=config, top_k=self.top_k)
+        pretrained_s1 = config.get("pretrained_s1")
+        if pretrained_s1 and is_train:
+            # print(self.load_state_dict(torch.load(pretrained_s1,map_location="cpu")["state_dict"]))
+            print(
+                self.load_state_dict(
+                    torch.load(pretrained_s1, map_location="cpu")["weight"]
+                )
+            )
+        if is_train:
+            self.automatic_optimization = False
+            self.save_hyperparameters()
+            self.eval_dir = output_dir / "eval"
+            self.eval_dir.mkdir(parents=True, exist_ok=True)
+
+    def training_step(self, batch: Dict, batch_idx: int):
+        opt = self.optimizers()
+        scheduler = self.lr_schedulers()
+        loss, acc = self.model.forward(
+            batch["phoneme_ids"],
+            batch["phoneme_ids_len"],
+            batch["semantic_ids"],
+            batch["semantic_ids_len"],
+            batch["bert_feature"],
+        )
+        self.manual_backward(loss)
+        if batch_idx > 0 and batch_idx % 4 == 0:
+            opt.step()
+            opt.zero_grad()
+            scheduler.step()
+
+        self.log(
+            "total_loss",
+            loss,
+            on_step=True,
+            on_epoch=True,
+            prog_bar=True,
+            sync_dist=True,
+        )
+        self.log(
+            "lr",
+            scheduler.get_last_lr()[0],
+            on_epoch=True,
+            prog_bar=True,
+            sync_dist=True,
+        )
+        self.log(
+            f"top_{self.top_k}_acc",
+            acc,
+            on_step=True,
+            on_epoch=True,
+            prog_bar=True,
+            sync_dist=True,
+        )
+
+    def validation_step(self, batch: Dict, batch_idx: int):
+        return
+
+    def configure_optimizers(self):
+        model_parameters = self.model.parameters()
+        parameters_names = []
+        parameters_names.append(
+            [name_param_pair[0] for name_param_pair in self.model.named_parameters()]
+        )
+        lm_opt = ScaledAdam(
+            model_parameters,
+            lr=0.01,
+            betas=(0.9, 0.95),
+            clipping_scale=2.0,
+            parameters_names=parameters_names,
+            show_dominant_parameters=False,
+            clipping_update_period=1000,
+        )
+
+        return {
+            "optimizer": lm_opt,
+            "lr_scheduler": {
+                "scheduler": WarmupCosineLRSchedule(
+                    lm_opt,
+                    init_lr=self.config["optimizer"]["lr_init"],
+                    peak_lr=self.config["optimizer"]["lr"],
+                    end_lr=self.config["optimizer"]["lr_end"],
+                    warmup_steps=self.config["optimizer"]["warmup_steps"],
+                    total_steps=self.config["optimizer"]["decay_steps"],
+                )
+            },
+        }

GPT_SoVITS/AR/models/t2s_model.py

@@ -0,0 +1,327 @@
+# modified from https://github.com/feng-yufei/shared_debugging_code/blob/main/model/t2s_model.py
+import torch
+from tqdm import tqdm
+
+from AR.models.utils import make_pad_mask
+from AR.models.utils import (
+    topk_sampling,
+    sample,
+    logits_to_probs,
+    multinomial_sample_one_no_sync,
+)
+from AR.modules.embedding import SinePositionalEmbedding
+from AR.modules.embedding import TokenEmbedding
+from AR.modules.transformer import LayerNorm
+from AR.modules.transformer import TransformerEncoder
+from AR.modules.transformer import TransformerEncoderLayer
+from torch import nn
+from torch.nn import functional as F
+from torchmetrics.classification import MulticlassAccuracy
+
+default_config = {
+    "embedding_dim": 512,
+    "hidden_dim": 512,
+    "num_head": 8,
+    "num_layers": 12,
+    "num_codebook": 8,
+    "p_dropout": 0.0,
+    "vocab_size": 1024 + 1,
+    "phoneme_vocab_size": 512,
+    "EOS": 1024,
+}
+
+
+class Text2SemanticDecoder(nn.Module):
+    def __init__(self, config, norm_first=False, top_k=3):
+        super(Text2SemanticDecoder, self).__init__()
+        self.model_dim = config["model"]["hidden_dim"]
+        self.embedding_dim = config["model"]["embedding_dim"]
+        self.num_head = config["model"]["head"]
+        self.num_layers = config["model"]["n_layer"]
+        self.norm_first = norm_first
+        self.vocab_size = config["model"]["vocab_size"]
+        self.phoneme_vocab_size = config["model"]["phoneme_vocab_size"]
+        self.p_dropout = config["model"]["dropout"]
+        self.EOS = config["model"]["EOS"]
+        self.norm_first = norm_first
+        assert self.EOS == self.vocab_size - 1
+        # should be same as num of kmeans bin
+        # assert self.EOS == 1024
+        self.bert_proj = nn.Linear(1024, self.embedding_dim)
+        self.ar_text_embedding = TokenEmbedding(
+            self.embedding_dim, self.phoneme_vocab_size, self.p_dropout
+        )
+        self.ar_text_position = SinePositionalEmbedding(
+            self.embedding_dim, dropout=0.1, scale=False, alpha=True
+        )
+        self.ar_audio_embedding = TokenEmbedding(
+            self.embedding_dim, self.vocab_size, self.p_dropout
+        )
+        self.ar_audio_position = SinePositionalEmbedding(
+            self.embedding_dim, dropout=0.1, scale=False, alpha=True
+        )
+
+        self.h = TransformerEncoder(
+            TransformerEncoderLayer(
+                d_model=self.model_dim,
+                nhead=self.num_head,
+                dim_feedforward=self.model_dim * 4,
+                dropout=0.1,
+                batch_first=True,
+                norm_first=norm_first,
+            ),
+            num_layers=self.num_layers,
+            norm=LayerNorm(self.model_dim) if norm_first else None,
+        )
+
+        self.ar_predict_layer = nn.Linear(self.model_dim, self.vocab_size, bias=False)
+        self.loss_fct = nn.CrossEntropyLoss(reduction="sum")
+
+        self.ar_accuracy_metric = MulticlassAccuracy(
+            self.vocab_size,
+            top_k=top_k,
+            average="micro",
+            multidim_average="global",
+            ignore_index=self.EOS,
+        )
+
+    def forward(self, x, x_lens, y, y_lens, bert_feature):
+        """
+        x: phoneme_ids
+        y: semantic_ids
+        """
+        x = self.ar_text_embedding(x)
+        x = x + self.bert_proj(bert_feature.transpose(1, 2))
+        x = self.ar_text_position(x)
+        x_mask = make_pad_mask(x_lens)
+
+        y_mask = make_pad_mask(y_lens)
+        y_mask_int = y_mask.type(torch.int64)
+        codes = y.type(torch.int64) * (1 - y_mask_int)
+
+        # Training
+        # AR Decoder
+        y, targets = self.pad_y_eos(codes, y_mask_int, eos_id=self.EOS)
+        x_len = x_lens.max()
+        y_len = y_lens.max()
+        y_emb = self.ar_audio_embedding(y)
+        y_pos = self.ar_audio_position(y_emb)
+
+        xy_padding_mask = torch.concat([x_mask, y_mask], dim=1)
+        ar_xy_padding_mask = xy_padding_mask
+
+        x_attn_mask = F.pad(
+            torch.zeros((x_len, x_len), dtype=torch.bool, device=x.device),
+            (0, y_len),
+            value=True,
+        )
+        y_attn_mask = F.pad(
+            torch.triu(
+                torch.ones(y_len, y_len, dtype=torch.bool, device=x.device),
+                diagonal=1,
+            ),
+            (x_len, 0),
+            value=False,
+        )
+        xy_attn_mask = torch.concat([x_attn_mask, y_attn_mask], dim=0)
+        bsz, src_len = x.shape[0], x_len + y_len
+        _xy_padding_mask = (
+            ar_xy_padding_mask.view(bsz, 1, 1, src_len)
+            .expand(-1, self.num_head, -1, -1)
+            .reshape(bsz * self.num_head, 1, src_len)
+        )
+        xy_attn_mask = xy_attn_mask.logical_or(_xy_padding_mask)
+        new_attn_mask = torch.zeros_like(xy_attn_mask, dtype=x.dtype)
+        new_attn_mask.masked_fill_(xy_attn_mask, float("-inf"))
+        xy_attn_mask = new_attn_mask
+        # x 和完整的 y 一次性输入模型
+        xy_pos = torch.concat([x, y_pos], dim=1)
+        xy_dec, _ = self.h(
+            (xy_pos, None),
+            mask=xy_attn_mask,
+        )
+        logits = self.ar_predict_layer(xy_dec[:, x_len:]).permute(0, 2, 1)
+        # loss
+        # from feiteng: 每次 duration 越多, 梯度更新也应该更多, 所以用 sum
+        loss = F.cross_entropy(logits, targets, reduction="sum")
+        acc = self.ar_accuracy_metric(logits.detach(), targets).item()
+        return loss, acc
+
+    # 需要看下这个函数和 forward 的区别以及没有 semantic 的时候 prompts 输入什么
+    def infer(
+        self,
+        x,
+        x_lens,
+        prompts,
+        bert_feature,
+        top_k: int = -100,
+        early_stop_num: int = -1,
+        temperature: float = 1.0,
+    ):
+        x = self.ar_text_embedding(x)
+        x = x + self.bert_proj(bert_feature.transpose(1, 2))
+        x = self.ar_text_position(x)
+
+        # AR Decoder
+        y = prompts
+        prefix_len = y.shape[1]
+        x_len = x.shape[1]
+        x_attn_mask = torch.zeros((x_len, x_len), dtype=torch.bool)
+        stop = False
+        for _ in tqdm(range(1500)):
+            y_emb = self.ar_audio_embedding(y)
+            y_pos = self.ar_audio_position(y_emb)
+            # x 和逐渐增长的 y 一起输入给模型
+            xy_pos = torch.concat([x, y_pos], dim=1)
+            y_len = y.shape[1]
+            x_attn_mask_pad = F.pad(
+                x_attn_mask,
+                (0, y_len),
+                value=True,
+            )
+            y_attn_mask = F.pad(
+                torch.triu(torch.ones(y_len, y_len, dtype=torch.bool), diagonal=1),
+                (x_len, 0),
+                value=False,
+            )
+            xy_attn_mask = torch.concat([x_attn_mask_pad, y_attn_mask], dim=0).to(
+                y.device
+            )
+
+            xy_dec, _ = self.h(
+                (xy_pos, None),
+                mask=xy_attn_mask,
+            )
+            logits = self.ar_predict_layer(xy_dec[:, -1])
+            samples = topk_sampling(
+                logits, top_k=top_k, top_p=1.0, temperature=temperature
+            )
+
+            if early_stop_num != -1 and (y.shape[1] - prefix_len) > early_stop_num:
+                print("use early stop num:", early_stop_num)
+                stop = True
+
+            if torch.argmax(logits, dim=-1)[0] == self.EOS or samples[0, 0] == self.EOS:
+                # print(torch.argmax(logits, dim=-1)[0] == self.EOS, samples[0, 0] == self.EOS)
+                stop = True
+            if stop:
+                if prompts.shape[1] == y.shape[1]:
+                    y = torch.concat([y, torch.zeros_like(samples)], dim=1)
+                    print("bad zero prediction")
+                print(f"T2S Decoding EOS [{prefix_len} -> {y.shape[1]}]")
+                break
+            # 本次生成的 semantic_ids 和之前的 y 构成新的 y
+            # print(samples.shape)#[1,1]#第一个1是bs
+            # import os
+            # os._exit(2333)
+            y = torch.concat([y, samples], dim=1)
+        return y
+
+    def pad_y_eos(self, y, y_mask_int, eos_id):
+        targets = F.pad(y, (0, 1), value=0) + eos_id * F.pad(
+            y_mask_int, (0, 1), value=1
+        )
+        # 错位
+        return targets[:, :-1], targets[:, 1:]
+
+    def infer_panel(
+        self,
+        x,  #####全部文本token
+        x_lens,
+        prompts,  ####参考音频token
+        bert_feature,
+        top_k: int = -100,
+        early_stop_num: int = -1,
+        temperature: float = 1.0,
+    ):
+        x = self.ar_text_embedding(x)
+        x = x + self.bert_proj(bert_feature.transpose(1, 2))
+        x = self.ar_text_position(x)
+
+        # AR Decoder
+        y = prompts
+        prefix_len = y.shape[1]
+        x_len = x.shape[1]
+        x_attn_mask = torch.zeros((x_len, x_len), dtype=torch.bool)
+        stop = False
+        # print(1111111,self.num_layers)
+        cache = {
+            "all_stage": self.num_layers,
+            "k": [None] * self.num_layers,  ###根据配置自己手写
+            "v": [None] * self.num_layers,
+            # "xy_pos":None,##y_pos位置编码每次都不一样的没法缓存，每次都要重新拼xy_pos.主要还是写法原因，其实是可以历史统一一样的，但也没啥计算量就不管了
+            "y_emb": None,  ##只需要对最新的samples求emb，再拼历史的就行
+            # "logits":None,###原版就已经只对结尾求再拼接了，不用管
+            # "xy_dec":None,###不需要，本来只需要最后一个做logits
+            "first_infer": 1,
+            "stage": 0,
+        }
+        for idx in tqdm(range(1500)):
+            if cache["first_infer"] == 1:
+                y_emb = self.ar_audio_embedding(y)
+            else:
+                y_emb = torch.cat(
+                    [cache["y_emb"], self.ar_audio_embedding(y[:, -1:])], 1
+                )
+            cache["y_emb"] = y_emb
+            y_pos = self.ar_audio_position(y_emb)
+            # x 和逐渐增长的 y 一起输入给模型
+            if cache["first_infer"] == 1:
+                xy_pos = torch.concat([x, y_pos], dim=1)
+            else:
+                xy_pos = y_pos[:, -1:]
+            y_len = y_pos.shape[1]
+            ###以下3个不做缓存
+            if cache["first_infer"] == 1:
+                x_attn_mask_pad = F.pad(
+                    x_attn_mask,
+                    (0, y_len),  ###xx的纯0扩展到xx纯0+xy纯1，(x,x+y)
+                    value=True,
+                )
+                y_attn_mask = F.pad(  ###yy的右上1扩展到左边xy的0,(y,x+y)
+                    torch.triu(torch.ones(y_len, y_len, dtype=torch.bool), diagonal=1),
+                    (x_len, 0),
+                    value=False,
+                )
+                xy_attn_mask = torch.concat([x_attn_mask_pad, y_attn_mask], dim=0).to(
+                    y.device
+                )
+            else:
+                ###最右边一列（是错的）
+                # xy_attn_mask=torch.ones((1, x_len+y_len), dtype=torch.bool,device=xy_pos.device)
+                # xy_attn_mask[:,-1]=False
+                ###最下面一行（是对的）
+                xy_attn_mask = torch.zeros(
+                    (1, x_len + y_len), dtype=torch.bool, device=xy_pos.device
+                )
+            # pdb.set_trace()
+            ###缓存重头戏
+            # print(1111,xy_pos.shape,xy_attn_mask.shape,x_len,y_len)
+            xy_dec, _ = self.h((xy_pos, None), mask=xy_attn_mask, cache=cache)
+            logits = self.ar_predict_layer(
+                xy_dec[:, -1]
+            )  ##不用改，如果用了cache的默认就是只有一帧，取最后一帧一样的
+            # samples = topk_sampling(logits, top_k=top_k, top_p=1.0, temperature=temperature)
+            if(idx==0):###第一次跑不能EOS否则没有了
+                logits = logits[:, :-1]  ###刨除1024终止符号的概率
+            samples = sample(
+                logits[0], y, top_k=top_k, top_p=1.0, repetition_penalty=1.35
+            )[0].unsqueeze(0)
+            if early_stop_num != -1 and (y.shape[1] - prefix_len) > early_stop_num:
+                print("use early stop num:", early_stop_num)
+                stop = True
+
+            if torch.argmax(logits, dim=-1)[0] == self.EOS or samples[0, 0] == self.EOS:
+                # print(torch.argmax(logits, dim=-1)[0] == self.EOS, samples[0, 0] == self.EOS)
+                stop = True
+            if stop:
+                if prompts.shape[1] == y.shape[1]:
+                    y = torch.concat([y, torch.zeros_like(samples)], dim=1)
+                    print("bad zero prediction")
+                print(f"T2S Decoding EOS [{prefix_len} -> {y.shape[1]}]")
+                break
+            # 本次生成的 semantic_ids 和之前的 y 构成新的 y
+            # print(samples.shape)#[1,1]#第一个1是bs
+            y = torch.concat([y, samples], dim=1)
+            cache["first_infer"] = 0
+        return y, idx

GPT_SoVITS/AR/models/t2s_model_onnx.py

@@ -0,0 +1,337 @@
+# modified from https://github.com/feng-yufei/shared_debugging_code/blob/main/model/t2s_model.py
+import torch
+from tqdm import tqdm
+
+from AR.modules.embedding_onnx import SinePositionalEmbedding
+from AR.modules.embedding_onnx import TokenEmbedding
+from AR.modules.transformer_onnx import LayerNorm
+from AR.modules.transformer_onnx import TransformerEncoder
+from AR.modules.transformer_onnx import TransformerEncoderLayer
+from torch import nn
+from torch.nn import functional as F
+from torchmetrics.classification import MulticlassAccuracy
+
+default_config = {
+    "embedding_dim": 512,
+    "hidden_dim": 512,
+    "num_head": 8,
+    "num_layers": 12,
+    "num_codebook": 8,
+    "p_dropout": 0.0,
+    "vocab_size": 1024 + 1,
+    "phoneme_vocab_size": 512,
+    "EOS": 1024,
+}
+
+inf_tensor_value = torch.FloatTensor([-float("Inf")]).float()
+
+def logits_to_probs(
+    logits,
+    previous_tokens = None,
+    temperature: float = 1.0,
+    top_k = None,
+    top_p = None,
+    repetition_penalty: float = 1.0,
+):
+    previous_tokens = previous_tokens.squeeze()
+    if previous_tokens is not None and repetition_penalty != 1.0:
+        previous_tokens = previous_tokens.long()
+        score = torch.gather(logits, dim=0, index=previous_tokens)
+        score = torch.where(
+            score < 0, score * repetition_penalty, score / repetition_penalty
+        )
+        logits.scatter_(dim=0, index=previous_tokens, src=score)
+
+    if top_p is not None and top_p < 1.0:
+        sorted_logits, sorted_indices = torch.sort(logits, descending=True)
+        cum_probs = torch.cumsum(
+            torch.nn.functional.softmax(sorted_logits, dim=-1), dim=-1
+        )
+        sorted_indices_to_remove = cum_probs > top_p
+        sorted_indices_to_remove[0] = False  # keep at least one option
+        indices_to_remove = sorted_indices_to_remove.scatter(
+            dim=0, index=sorted_indices, src=sorted_indices_to_remove
+        )
+        logits = logits.masked_fill(indices_to_remove, -float("Inf"))
+
+    logits = logits / max(temperature, 1e-5)
+
+    if top_k is not None:
+        v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
+        pivot = v.select(-1, -1).unsqueeze(-1)
+        logits = torch.where(logits < pivot, inf_tensor_value, logits)
+
+    probs = torch.nn.functional.softmax(logits, dim=-1)
+    return probs
+
+
+def multinomial_sample_one_no_sync(
+    probs_sort
+):  # Does multinomial sampling without a cuda synchronization
+    q = torch.randn_like(probs_sort)
+    return torch.argmax(probs_sort / q, dim=-1, keepdim=True).to(dtype=torch.int)
+
+
+def sample(
+    logits,
+    previous_tokens,
+    **sampling_kwargs,
+):
+    probs = logits_to_probs(
+        logits=logits, previous_tokens=previous_tokens, **sampling_kwargs
+    )
+    idx_next = multinomial_sample_one_no_sync(probs)
+    return idx_next, probs
+
+
+class OnnxEncoder(nn.Module):
+    def __init__(self, ar_text_embedding, bert_proj, ar_text_position):
+        super().__init__()
+        self.ar_text_embedding = ar_text_embedding
+        self.bert_proj = bert_proj
+        self.ar_text_position = ar_text_position
+    
+    def forward(self, x, bert_feature):
+        x = self.ar_text_embedding(x)
+        x = x + self.bert_proj(bert_feature.transpose(1, 2))
+        return self.ar_text_position(x)
+
+
+class T2SFirstStageDecoder(nn.Module):
+    def __init__(self, ar_audio_embedding, ar_audio_position, h, ar_predict_layer, loss_fct, ar_accuracy_metric,
+    top_k, early_stop_num, num_layers):
+        super().__init__()
+        self.ar_audio_embedding = ar_audio_embedding
+        self.ar_audio_position = ar_audio_position
+        self.h = h
+        self.ar_predict_layer = ar_predict_layer
+        self.loss_fct = loss_fct
+        self.ar_accuracy_metric = ar_accuracy_metric
+        self.top_k = top_k
+        self.early_stop_num = early_stop_num
+        self.num_layers = num_layers
+    
+    def forward(self, x, prompt):
+        y = prompt
+        x_example = x[:,:,0] * 0.0
+        #N, 1, 512
+        cache = {
+            "all_stage": self.num_layers,
+            "k": None,
+            "v": None,
+            "y_emb": None,
+            "first_infer": 1,
+            "stage": 0,
+        }
+
+        y_emb = self.ar_audio_embedding(y)
+
+        cache["y_emb"] = y_emb
+        y_pos = self.ar_audio_position(y_emb)
+
+        xy_pos = torch.concat([x, y_pos], dim=1)
+
+        y_example = y_pos[:,:,0] * 0.0
+        x_attn_mask = torch.matmul(x_example.transpose(0, 1) , x_example).bool()
+        y_attn_mask = torch.ones_like(torch.matmul(y_example.transpose(0, 1), y_example), dtype=torch.int64)
+        y_attn_mask = torch.cumsum(y_attn_mask, dim=1) - torch.cumsum(
+            torch.ones_like(y_example.transpose(0, 1), dtype=torch.int64), dim=0
+        )
+        y_attn_mask = y_attn_mask > 0
+
+        x_y_pad = torch.matmul(x_example.transpose(0, 1), y_example).bool()
+        y_x_pad = torch.matmul(y_example.transpose(0, 1), x_example).bool()
+        x_attn_mask_pad = torch.cat([x_attn_mask, torch.ones_like(x_y_pad)], dim=1)
+        y_attn_mask = torch.cat([y_x_pad, y_attn_mask], dim=1)
+        xy_attn_mask = torch.concat([x_attn_mask_pad, y_attn_mask], dim=0)
+        cache["k"] = torch.matmul(x_attn_mask_pad[0].float().unsqueeze(-1), torch.zeros((1, 512)))\
+        .unsqueeze(1).repeat(self.num_layers, 1, 1, 1)
+        cache["v"] = torch.matmul(x_attn_mask_pad[0].float().unsqueeze(-1), torch.zeros((1, 512)))\
+        .unsqueeze(1).repeat(self.num_layers, 1, 1, 1)
+
+        xy_dec = self.h(xy_pos, mask=xy_attn_mask, cache=cache)
+        logits = self.ar_predict_layer(xy_dec[:, -1])
+        samples = sample(logits[0], y, top_k=self.top_k, top_p=1.0, repetition_penalty=1.35)[0].unsqueeze(0)
+
+        y = torch.concat([y, samples], dim=1)
+
+        return y, cache["k"], cache["v"], cache["y_emb"], x_example
+
+
+class T2SStageDecoder(nn.Module):
+    def __init__(self, ar_audio_embedding, ar_audio_position, h, ar_predict_layer, loss_fct, ar_accuracy_metric,
+    top_k, early_stop_num, num_layers):
+        super().__init__()
+        self.ar_audio_embedding = ar_audio_embedding
+        self.ar_audio_position = ar_audio_position
+        self.h = h
+        self.ar_predict_layer = ar_predict_layer
+        self.loss_fct = loss_fct
+        self.ar_accuracy_metric = ar_accuracy_metric
+        self.top_k = top_k
+        self.early_stop_num = early_stop_num
+        self.num_layers = num_layers
+
+    def forward(self, y, k, v, y_emb, x_example):
+        cache = {
+            "all_stage": self.num_layers,
+            "k": torch.nn.functional.pad(k, (0, 0, 0, 0, 0, 1)),
+            "v": torch.nn.functional.pad(v, (0, 0, 0, 0, 0, 1)),
+            "y_emb": y_emb,
+            "first_infer": 0,
+            "stage": 0,
+        }
+
+        y_emb = torch.cat(
+            [cache["y_emb"], self.ar_audio_embedding(y[:, -1:])], 1
+        )
+        cache["y_emb"] = y_emb
+        y_pos = self.ar_audio_position(y_emb)
+
+        xy_pos = y_pos[:, -1:]
+        
+        y_example = y_pos[:,:,0] * 0.0
+
+        xy_attn_mask = torch.cat([x_example, y_example], dim=1)
+        xy_attn_mask = torch.zeros_like(xy_attn_mask, dtype=torch.bool)
+
+        xy_dec = self.h(xy_pos, mask=xy_attn_mask, cache=cache)
+        logits = self.ar_predict_layer(xy_dec[:, -1])
+        samples = sample(logits[0], y, top_k=self.top_k, top_p=1.0, repetition_penalty=1.35)[0].unsqueeze(0)
+
+        y = torch.concat([y, samples], dim=1)
+
+        return y, cache["k"], cache["v"], cache["y_emb"], logits, samples
+
+
+class Text2SemanticDecoder(nn.Module):
+    def __init__(self, config, norm_first=False, top_k=3):
+        super(Text2SemanticDecoder, self).__init__()
+        self.model_dim = config["model"]["hidden_dim"]
+        self.embedding_dim = config["model"]["embedding_dim"]
+        self.num_head = config["model"]["head"]
+        self.num_layers = config["model"]["n_layer"]
+        self.norm_first = norm_first
+        self.vocab_size = config["model"]["vocab_size"]
+        self.phoneme_vocab_size = config["model"]["phoneme_vocab_size"]
+        self.p_dropout = float(config["model"]["dropout"])
+        self.EOS = config["model"]["EOS"]
+        self.norm_first = norm_first
+        assert self.EOS == self.vocab_size - 1
+        self.bert_proj = nn.Linear(1024, self.embedding_dim)
+        self.ar_text_embedding = TokenEmbedding(self.embedding_dim, self.phoneme_vocab_size, self.p_dropout)
+        self.ar_text_position = SinePositionalEmbedding(self.embedding_dim, dropout=0.1, scale=False, alpha=True)
+        self.ar_audio_embedding = TokenEmbedding(self.embedding_dim, self.vocab_size, self.p_dropout)
+        self.ar_audio_position = SinePositionalEmbedding(self.embedding_dim, dropout=0.1, scale=False, alpha=True)
+        self.h = TransformerEncoder(
+            TransformerEncoderLayer(
+                d_model=self.model_dim,
+                nhead=self.num_head,
+                dim_feedforward=self.model_dim * 4,
+                dropout=0.1,
+                batch_first=True,
+                norm_first=norm_first,
+            ),
+            num_layers=self.num_layers,
+            norm=LayerNorm(self.model_dim) if norm_first else None,
+        )
+        self.ar_predict_layer = nn.Linear(self.model_dim, self.vocab_size, bias=False)
+        self.loss_fct = nn.CrossEntropyLoss(reduction="sum")
+        self.ar_accuracy_metric = MulticlassAccuracy(
+            self.vocab_size,
+            top_k=top_k,
+            average="micro",
+            multidim_average="global",
+            ignore_index=self.EOS,
+        )
+        self.top_k = torch.LongTensor([1])
+        self.early_stop_num = torch.LongTensor([-1])
+
+    def init_onnx(self):
+        self.onnx_encoder = OnnxEncoder(self.ar_text_embedding, self.bert_proj, self.ar_text_position)
+        self.first_stage_decoder = T2SFirstStageDecoder(self.ar_audio_embedding, self.ar_audio_position, self.h, 
+            self.ar_predict_layer, self.loss_fct, self.ar_accuracy_metric, self.top_k, self.early_stop_num,
+            self.num_layers)
+        self.stage_decoder = T2SStageDecoder(self.ar_audio_embedding, self.ar_audio_position, self.h, 
+            self.ar_predict_layer, self.loss_fct, self.ar_accuracy_metric, self.top_k, self.early_stop_num,
+            self.num_layers)
+
+    def forward(self, x, prompts, bert_feature):
+        early_stop_num = self.early_stop_num
+        prefix_len = prompts.shape[1]
+
+        x = self.onnx_encoder(x, bert_feature)
+        y, k, v, y_emb, stage, x_example = self.first_stage_decoder(x, prompts)
+
+        stop = False
+        for idx in range(1, 1500):
+            enco = self.stage_decoder(y, k, v, y_emb, stage, x_example)
+            y, k, v, y_emb, stage, logits, samples = enco
+            if early_stop_num != -1 and (y.shape[1] - prefix_len) > early_stop_num:
+                stop = True
+            if torch.argmax(logits, dim=-1)[0] == self.EOS or samples[0, 0] == self.EOS:
+                stop = True
+            if stop:
+                break
+        y[0, -1] = 0
+        return y, idx
+
+    def infer(self, x, prompts, bert_feature):
+        top_k = self.top_k
+        early_stop_num = self.early_stop_num
+
+        x = self.onnx_encoder(x, bert_feature)
+
+        y = prompts
+        prefix_len = y.shape[1]
+        x_len = x.shape[1]
+        x_example = x[:,:,0] * 0.0
+        x_attn_mask = torch.matmul(x_example.transpose(0, 1), x_example)
+        x_attn_mask = torch.zeros_like(x_attn_mask, dtype=torch.bool)
+
+        stop = False
+        cache = {
+            "all_stage": self.num_layers,
+            "k": [None] * self.num_layers,
+            "v": [None] * self.num_layers,
+            "y_emb": None,
+            "first_infer": 1,
+            "stage": 0,
+        }
+        for idx in range(1500):
+            if cache["first_infer"] == 1:
+                y_emb = self.ar_audio_embedding(y)
+            else:
+                y_emb = torch.cat(
+                    [cache["y_emb"], self.ar_audio_embedding(y[:, -1:])], 1
+                )
+            cache["y_emb"] = y_emb
+            y_pos = self.ar_audio_position(y_emb)
+            if cache["first_infer"] == 1:
+                xy_pos = torch.concat([x, y_pos], dim=1)
+            else:
+                xy_pos = y_pos[:, -1:]
+            y_len = y_pos.shape[1]
+            if cache["first_infer"] == 1:
+                x_attn_mask_pad = F.pad(x_attn_mask, (0, y_len), value=True)
+                y_attn_mask = F.pad(
+                    torch.triu(torch.ones(y_len, y_len, dtype=torch.bool), diagonal=1),
+                    (x_len, 0), value=False
+                )
+                xy_attn_mask = torch.concat([x_attn_mask_pad, y_attn_mask], dim=0)
+            else:
+                xy_attn_mask = torch.zeros((1, x_len + y_len), dtype=torch.bool)
+            xy_dec = self.h(xy_pos, mask=xy_attn_mask, cache=cache)
+            logits = self.ar_predict_layer(xy_dec[:, -1])
+            samples = sample(logits[0], y, top_k=top_k, top_p=1.0, repetition_penalty=1.35)[0].unsqueeze(0)
+            if early_stop_num != -1 and (y.shape[1] - prefix_len) > early_stop_num:
+                stop = True
+            if torch.argmax(logits, dim=-1)[0] == self.EOS or samples[0, 0] == self.EOS:
+                stop = True
+            if stop:
+                if prompts.shape[1] == y.shape[1]:
+                    y = torch.concat([y, torch.zeros_like(samples)], dim=1)
+                break
+            y = torch.concat([y, samples], dim=1)
+            cache["first_infer"] = 0
+        return y, idx

GPT_SoVITS/AR/models/utils.py

@@ -0,0 +1,160 @@
+# modified from https://github.com/feng-yufei/shared_debugging_code/blob/main/model/utils.py\
+import torch
+import torch.nn.functional as F
+
+
+def sequence_mask(length, max_length=None):
+    if max_length is None:
+        max_length = length.max()
+    x = torch.arange(max_length, dtype=length.dtype, device=length.device)
+    return x.unsqueeze(0) < length.unsqueeze(1)
+
+
+def make_pad_mask(lengths: torch.Tensor, max_len: int = 0) -> torch.Tensor:
+    """
+    Args:
+      lengths:
+        A 1-D tensor containing sentence lengths.
+      max_len:
+        The length of masks.
+    Returns:
+      Return a 2-D bool tensor, where masked positions
+      are filled with `True` and non-masked positions are
+      filled with `False`.
+
+    #>>> lengths = torch.tensor([1, 3, 2, 5])
+    #>>> make_pad_mask(lengths)
+    tensor([[False,  True,  True,  True,  True],
+            [False, False, False,  True,  True],
+            [False, False,  True,  True,  True],
+            [False, False, False, False, False]])
+    """
+    assert lengths.ndim == 1, lengths.ndim
+    max_len = max(max_len, lengths.max())
+    n = lengths.size(0)
+    seq_range = torch.arange(0, max_len, device=lengths.device)
+    expaned_lengths = seq_range.unsqueeze(0).expand(n, max_len)
+
+    return expaned_lengths >= lengths.unsqueeze(-1)
+
+
+# https://github.com/microsoft/unilm/blob/master/xtune/src/transformers/modeling_utils.py
+def top_k_top_p_filtering(
+    logits, top_k=0, top_p=1.0, filter_value=-float("Inf"), min_tokens_to_keep=1
+):
+    """Filter a distribution of logits using top-k and/or nucleus (top-p) filtering
+    Args:
+        logits: logits distribution shape (batch size, vocabulary size)
+        if top_k > 0: keep only top k tokens with highest probability (top-k filtering).
+        if top_p < 1.0: keep the top tokens with cumulative probability >= top_p (nucleus filtering).
+            Nucleus filtering is described in Holtzman et al. (http://arxiv.org/abs/1904.09751)
+        Make sure we keep at least min_tokens_to_keep per batch example in the output
+    From: https://gist.github.com/thomwolf/1a5a29f6962089e871b94cbd09daf317
+    """
+    if top_k > 0:
+        top_k = min(max(top_k, min_tokens_to_keep), logits.size(-1))  # Safety check
+        # Remove all tokens with a probability less than the last token of the top-k
+        indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]
+        logits[indices_to_remove] = filter_value
+
+    if top_p < 1.0:
+        sorted_logits, sorted_indices = torch.sort(logits, descending=True)
+        cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
+
+        # Remove tokens with cumulative probability above the threshold (token with 0 are kept)
+        sorted_indices_to_remove = cumulative_probs > top_p
+        if min_tokens_to_keep > 1:
+            # Keep at least min_tokens_to_keep (set to min_tokens_to_keep-1 because we add the first one below)
+            sorted_indices_to_remove[..., :min_tokens_to_keep] = 0
+        # Shift the indices to the right to keep also the first token above the threshold
+        sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
+        sorted_indices_to_remove[..., 0] = 0
+
+        # scatter sorted tensors to original indexing
+        indices_to_remove = sorted_indices_to_remove.scatter(
+            1, sorted_indices, sorted_indices_to_remove
+        )
+        logits[indices_to_remove] = filter_value
+    return logits
+
+
+def topk_sampling(logits, top_k=10, top_p=1.0, temperature=1.0):
+    # temperature: (`optional`) float
+    #     The value used to module the next token probabilities. Must be strictly positive. Default to 1.0.
+    # top_k: (`optional`) int
+    #     The number of highest probability vocabulary tokens to keep for top-k-filtering. Between 1 and infinity. Default to 50.
+    # top_p: (`optional`) float
+    #     The cumulative probability of parameter highest probability vocabulary tokens to keep for nucleus sampling. Must be between 0 and 1. Default to 1.
+
+    # Temperature (higher temperature => more likely to sample low probability tokens)
+    if temperature != 1.0:
+        logits = logits / temperature
+    # Top-p/top-k filtering
+    logits = top_k_top_p_filtering(logits, top_k=top_k, top_p=top_p)
+    # Sample
+    token = torch.multinomial(F.softmax(logits, dim=-1), num_samples=1)
+    return token
+
+
+from typing import Optional, Tuple
+
+
+def multinomial_sample_one_no_sync(
+    probs_sort,
+):  # Does multinomial sampling without a cuda synchronization
+    q = torch.empty_like(probs_sort).exponential_(1)
+    return torch.argmax(probs_sort / q, dim=-1, keepdim=True).to(dtype=torch.int)
+
+
+def logits_to_probs(
+    logits,
+    previous_tokens: Optional[torch.Tensor] = None,
+    temperature: float = 1.0,
+    top_k: Optional[int] = None,
+    top_p: Optional[int] = None,
+    repetition_penalty: float = 1.0,
+):
+    previous_tokens = previous_tokens.squeeze()
+    # print(logits.shape,previous_tokens.shape)
+    # pdb.set_trace()
+    if previous_tokens is not None and repetition_penalty != 1.0:
+        previous_tokens = previous_tokens.long()
+        score = torch.gather(logits, dim=0, index=previous_tokens)
+        score = torch.where(
+            score < 0, score * repetition_penalty, score / repetition_penalty
+        )
+        logits.scatter_(dim=0, index=previous_tokens, src=score)
+
+    if top_p is not None and top_p < 1.0:
+        sorted_logits, sorted_indices = torch.sort(logits, descending=True)
+        cum_probs = torch.cumsum(
+            torch.nn.functional.softmax(sorted_logits, dim=-1), dim=-1
+        )
+        sorted_indices_to_remove = cum_probs > top_p
+        sorted_indices_to_remove[0] = False  # keep at least one option
+        indices_to_remove = sorted_indices_to_remove.scatter(
+            dim=0, index=sorted_indices, src=sorted_indices_to_remove
+        )
+        logits = logits.masked_fill(indices_to_remove, -float("Inf"))
+
+    logits = logits / max(temperature, 1e-5)
+
+    if top_k is not None:
+        v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
+        pivot = v.select(-1, -1).unsqueeze(-1)
+        logits = torch.where(logits < pivot, -float("Inf"), logits)
+
+    probs = torch.nn.functional.softmax(logits, dim=-1)
+    return probs
+
+
+def sample(
+    logits,
+    previous_tokens: Optional[torch.Tensor] = None,
+    **sampling_kwargs,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    probs = logits_to_probs(
+        logits=logits, previous_tokens=previous_tokens, **sampling_kwargs
+    )
+    idx_next = multinomial_sample_one_no_sync(probs)
+    return idx_next, probs