update

.gitignore

@@ -3,3 +3,7 @@
 /logs/
 /cache/
 /upload/
+/vits/text/chinese_dialect_lexicons/
+/bert_vits2/bert/chinese-roberta-wwm-ext-large/pytorch_model.bin
+/bert_vits2/bert/bert-base-japanese-v3/pytorch_model.bin
+/vits/bert/prosody_model.pt

README_zh.md

@@ -5,7 +5,7 @@
     <br/>
     <p>
         <img src="https://img.shields.io/github/license/Artrajz/vits-simple-api">
-    	<img src="https://img.shields.io/badge/python-3.9%7C3.10-green">
+    	<img src="https://img.shields.io/badge/python-3.10-green">
         <a href="https://hub.docker.com/r/artrajz/vits-simple-api">
             <img src="https://img.shields.io/docker/pulls/artrajz/vits-simple-api"></a>
     </p>
@@ -16,6 +16,7 @@
 
 
 
+
 # Feature
 
 - [x] VITS语音合成，语音转换
@@ -48,7 +49,7 @@ https://user-images.githubusercontent.com/73542220/237995061-c1f25b4e-dd86-438a-
 
 # 部署
 
-## Docker部署
+## Docker部署（Linux推荐）
 
 ### 镜像拉取脚本
 
@@ -56,7 +57,7 @@ https://user-images.githubusercontent.com/73542220/237995061-c1f25b4e-dd86-438a-
 bash -c "$(wget -O- https://raw.githubusercontent.com/Artrajz/vits-simple-api/main/vits-simple-api-installer-latest.sh)"
 ```
 
-- 目前docker镜像支持的平台`linux/amd64,linux/arm64`
+- 目前docker镜像支持的平台`linux/amd64,linux/arm64`（arm64仅有CPU版本）
 - 在拉取完成后，需要导入VITS模型才能使用，请根据以下步骤导入模型。
 
 ### 下载VITS模型
@@ -137,17 +138,15 @@ DIMENSIONAL_EMOTION_MODEL = ABS_PATH + "/Model/model.yaml"
 
 ###  下载python依赖
 
-推荐使用python的虚拟环境，python版本 >= 3.9
+推荐使用python的虚拟环境
 
 `pip install -r requirements.txt`
 
 windows下可能安装不了fasttext,可以用以下命令安装，附[wheels下载地址](https://www.lfd.uci.edu/~gohlke/pythonlibs/#fasttext)
 
 ```
-#python3.10 win_amd64
+# python3.10 win_amd64
 pip install https://github.com/Artrajz/archived/raw/main/fasttext/fasttext-0.9.2-cp310-cp310-win_amd64.whl
-#python3.9 win_amd64
-pip install https://github.com/Artrajz/archived/raw/main/fasttext/fasttext-0.9.2-cp39-cp39-win_amd64.whl
 ```
 
 ### 下载VITS模型
@@ -222,28 +221,24 @@ nvidia-smi
 
 ### 安装GPU版pytorch
 
-CUDA11.7对应的pytorch是用这个命令安装
+CUDA11.7对应的pytorch是用这个命令安装，推荐使用1.13.1+cu117，其他版本可能存在内存不稳定的问题。
 
 ```
-pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu117
+pip install torch==1.13.1+cu117 --extra-index-url https://download.pytorch.org/whl/cu117
 ```
 
-对应版本的命令可以在[官网](https://pytorch.org/get-started/locally/)找到
-
 ## Linux
 
 安装过程类似，但我没有相应的环境所以没办法测试
 
-# Openjtalk安装问题
+# 依赖安装问题
 
-如果你是arm64架构的平台，由于pypi官网上没有arm64对应的whl，可能安装会出现一些问题，你可以使用我构建的whl来安装
+由于pypi.org没有pyopenjtalk的whl文件，通常需要从源代码来安装，这一过程对于一些人来说可能比较麻烦，所以你也可以使用我构建的whl来安装。
 
 ```
-pip install openjtalk==0.3.0.dev2 --index-url https://pypi.artrajz.cn/simple
+pip install pyopenjtalk -i https://pypi.artrajz.cn/simple
 ```
 
-或者是自己手动构建一个whl，可以根据[教程](https://artrajz.cn/index.php/archives/167/)来构建
-
 # API
 
 ## GET
@@ -274,190 +269,9 @@ pip install openjtalk==0.3.0.dev2 --index-url https://pypi.artrajz.cn/simple
 
 ## POST
 
-- python
-
-```python
-import re
-import requests
-import os
-import random
-import string
-from requests_toolbelt.multipart.encoder import MultipartEncoder
-
-abs_path = os.path.dirname(__file__)
-base = "http://127.0.0.1:23456"
-
-
-# 映射表
-def voice_speakers():
-    url = f"{base}/voice/speakers"
-
-    res = requests.post(url=url)
-    json = res.json()
-    for i in json:
-        print(i)
-        for j in json[i]:
-            print(j)
-    return json
-
-
-# 语音合成 voice vits
-def voice_vits(text, id=0, format="wav", lang="auto", length=1, noise=0.667, noisew=0.8, max=50):
-    fields = {
-        "text": text,
-        "id": str(id),
-        "format": format,
-        "lang": lang,
-        "length": str(length),
-        "noise": str(noise),
-        "noisew": str(noisew),
-        "max": str(max)
-    }
-    boundary = '----VoiceConversionFormBoundary' + ''.join(random.sample(string.ascii_letters + string.digits, 16))
-
-    m = MultipartEncoder(fields=fields, boundary=boundary)
-    headers = {"Content-Type": m.content_type}
-    url = f"{base}/voice"
-
-    res = requests.post(url=url, data=m, headers=headers)
-    fname = re.findall("filename=(.+)", res.headers["Content-Disposition"])[0]
-    path = f"{abs_path}/{fname}"
-
-    with open(path, "wb") as f:
-        f.write(res.content)
-    print(path)
-    return path
-
-
-# 语音转换 hubert-vits
-def voice_hubert_vits(upload_path, id, format="wav", length=1, noise=0.667, noisew=0.8):
-    upload_name = os.path.basename(upload_path)
-    upload_type = f'audio/{upload_name.split(".")[1]}'  # wav,ogg
-
-    with open(upload_path, 'rb') as upload_file:
-        fields = {
-            "upload": (upload_name, upload_file, upload_type),
-            "id": str(id),
-            "format": format,
-            "length": str(length),
-            "noise": str(noise),
-            "noisew": str(noisew),
-        }
-        boundary = '----VoiceConversionFormBoundary' + ''.join(random.sample(string.ascii_letters + string.digits, 16))
-
-        m = MultipartEncoder(fields=fields, boundary=boundary)
-        headers = {"Content-Type": m.content_type}
-        url = f"{base}/voice/hubert-vits"
-
-        res = requests.post(url=url, data=m, headers=headers)
-    fname = re.findall("filename=(.+)", res.headers["Content-Disposition"])[0]
-    path = f"{abs_path}/{fname}"
-
-    with open(path, "wb") as f:
-        f.write(res.content)
-    print(path)
-    return path
-
-
-# 维度情感模型 w2v2-vits
-def voice_w2v2_vits(text, id=0, format="wav", lang="auto", length=1, noise=0.667, noisew=0.8, max=50, emotion=0):
-    fields = {
-        "text": text,
-        "id": str(id),
-        "format": format,
-        "lang": lang,
-        "length": str(length),
-        "noise": str(noise),
-        "noisew": str(noisew),
-        "max": str(max),
-        "emotion": str(emotion)
-    }
-    boundary = '----VoiceConversionFormBoundary' + ''.join(random.sample(string.ascii_letters + string.digits, 16))
-
-    m = MultipartEncoder(fields=fields, boundary=boundary)
-    headers = {"Content-Type": m.content_type}
-    url = f"{base}/voice/w2v2-vits"
-
-    res = requests.post(url=url, data=m, headers=headers)
-    fname = re.findall("filename=(.+)", res.headers["Content-Disposition"])[0]
-    path = f"{abs_path}/{fname}"
-
-    with open(path, "wb") as f:
-        f.write(res.content)
-    print(path)
-    return path
-
-
-# 语音转换 同VITS模型内角色之间的音色转换
-def voice_conversion(upload_path, original_id, target_id):
-    upload_name = os.path.basename(upload_path)
-    upload_type = f'audio/{upload_name.split(".")[1]}'  # wav,ogg
-
-    with open(upload_path, 'rb') as upload_file:
-        fields = {
-            "upload": (upload_name, upload_file, upload_type),
-            "original_id": str(original_id),
-            "target_id": str(target_id),
-        }
-        boundary = '----VoiceConversionFormBoundary' + ''.join(random.sample(string.ascii_letters + string.digits, 16))
-        m = MultipartEncoder(fields=fields, boundary=boundary)
-
-        headers = {"Content-Type": m.content_type}
-        url = f"{base}/voice/conversion"
-
-        res = requests.post(url=url, data=m, headers=headers)
-
-    fname = re.findall("filename=(.+)", res.headers["Content-Disposition"])[0]
-    path = f"{abs_path}/{fname}"
-
-    with open(path, "wb") as f:
-        f.write(res.content)
-    print(path)
-    return path
-
-
-def voice_ssml(ssml):
-    fields = {
-        "ssml": ssml,
-    }
-    boundary = '----VoiceConversionFormBoundary' + ''.join(random.sample(string.ascii_letters + string.digits, 16))
-
-    m = MultipartEncoder(fields=fields, boundary=boundary)
-    headers = {"Content-Type": m.content_type}
-    url = f"{base}/voice/ssml"
-
-    res = requests.post(url=url, data=m, headers=headers)
-    fname = re.findall("filename=(.+)", res.headers["Content-Disposition"])[0]
-    path = f"{abs_path}/{fname}"
-
-    with open(path, "wb") as f:
-        f.write(res.content)
-    print(path)
-    return path
-
-def voice_dimensional_emotion(upload_path):
-    upload_name = os.path.basename(upload_path)
-    upload_type = f'audio/{upload_name.split(".")[1]}'  # wav,ogg
-
-    with open(upload_path, 'rb') as upload_file:
-        fields = {
-            "upload": (upload_name, upload_file, upload_type),
-        }
-        boundary = '----VoiceConversionFormBoundary' + ''.join(random.sample(string.ascii_letters + string.digits, 16))
-
-        m = MultipartEncoder(fields=fields, boundary=boundary)
-        headers = {"Content-Type": m.content_type}
-        url = f"{base}/voice/dimension-emotion"
-
-        res = requests.post(url=url, data=m, headers=headers)
-    fname = re.findall("filename=(.+)", res.headers["Content-Disposition"])[0]
-    path = f"{abs_path}/{fname}"
-
-    with open(path, "wb") as f:
-        f.write(res.content)
-    print(path)
-    return path
-```
+- 见`api_test.py`
+
+
 
 ## API KEY
 
@@ -615,4 +429,5 @@ def voice_dimensional_emotion(upload_path):
 - emotional-vits:https://github.com/innnky/emotional-vits
 - vits-uma-genshin-honkai:https://huggingface.co/spaces/zomehwh/vits-uma-genshin-honkai
 - vits_chinese:https://github.com/PlayVoice/vits_chinese
+- Bert_VITS2:https://github.com/fishaudio/Bert-VITS2
 

api_test.py

@@ -0,0 +1,429 @@
+import json
+import re
+import requests
+import os
+import time
+import random
+import string
+from requests_toolbelt.multipart.encoder import MultipartEncoder
+
+absolute_path = os.path.dirname(__file__)
+base_url = "http://127.0.0.1:23456"
+
+
+# 映射表
+def voice_speakers():
+    url = f"{base_url}/voice/speakers"
+
+    res = requests.post(url=url)
+    json = res.json()
+    for i in json:
+        print(i)
+        for j in json[i]:
+            print(j)
+    return json
+
+
+# 语音合成 voice vits
+def voice_vits(text, id=0, format="wav", lang="auto", length=1, noise=0.667, noisew=0.8, max=50, save_audio=True,
+               save_path=None):
+    fields = {
+        "text": text,
+        "id": str(id),
+        "format": format,
+        "lang": lang,
+        "length": str(length),
+        "noise": str(noise),
+        "noisew": str(noisew),
+        "max": str(max)
+    }
+    boundary = '----VoiceConversionFormBoundary' + ''.join(random.sample(string.ascii_letters + string.digits, 16))
+
+    m = MultipartEncoder(fields=fields, boundary=boundary)
+    headers = {"Content-Type": m.content_type}
+    url = f"{base_url}/voice/vits"
+
+    res = requests.post(url=url, data=m, headers=headers)
+    fname = re.findall("filename=(.+)", res.headers["Content-Disposition"])[0]
+    if save_path is not None:
+        path = os.path.join(save_path, fname)
+    else:
+        path = os.path.join(absolute_path, fname)
+    if save_audio:
+        with open(path, "wb") as f:
+            f.write(res.content)
+        print(path)
+        return path
+    return None
+
+
+def voice_vits_streaming(text, id=0, format="wav", lang="auto", length=1, noise=0.667, noisew=0.8, max=50,
+                         save_audio=True, save_path=None):
+    fields = {
+        "text": text,
+        "id": str(id),
+        "format": format,
+        "lang": lang,
+        "length": str(length),
+        "noise": str(noise),
+        "noisew": str(noisew),
+        "max": str(max),
+        "streaming": 'True'
+    }
+    boundary = '----VoiceConversionFormBoundary' + ''.join(random.sample(string.ascii_letters + string.digits, 16))
+
+    m = MultipartEncoder(fields=fields, boundary=boundary)
+    headers = {"Content-Type": m.content_type}
+    url = f"{base_url}/voice"
+
+    res = requests.post(url=url, data=m, headers=headers)
+    fname = re.findall("filename=(.+)", res.headers["Content-Disposition"])[0]
+    if save_path is not None:
+        path = os.path.join(save_path, fname)
+    else:
+        path = os.path.join(absolute_path, fname)
+    if save_audio:
+        with open(path, "wb") as f:
+            f.write(res.content)
+        print(path)
+        return path
+    return None
+
+
+def voice_vits_streaming(text, id=0, format="wav", lang="auto", length=1, noise=0.667, noisew=0.8, max=50,
+                         save_path=None):
+    fields = {
+        "text": text,
+        "id": str(id),
+        "format": format,
+        "lang": lang,
+        "length": str(length),
+        "noise": str(noise),
+        "noisew": str(noisew),
+        "max": str(max),
+        "streaming": 'True'
+    }
+    boundary = '----VoiceConversionFormBoundary' + ''.join(random.sample(string.ascii_letters + string.digits, 16))
+
+    m = MultipartEncoder(fields=fields, boundary=boundary)
+    headers = {"Content-Type": m.content_type}
+    url = f"{base_url}/voice"
+
+    res = requests.post(url=url, data=m, headers=headers, stream=True)
+    fname = re.findall("filename=(.+)", res.headers["Content-Disposition"])[0]
+    if save_path is not None:
+        path = os.path.join(save_path, fname)
+    else:
+        path = os.path.join(absolute_path, fname)
+    audio = res.content
+
+    def get_file_size_from_bytes(byte_data):
+        file_size_offset = 4
+        file_size_length = 4
+
+        try:
+            file_size_bytes = byte_data[file_size_offset:file_size_offset + file_size_length]
+            file_size = int.from_bytes(file_size_bytes, byteorder='little')
+            return file_size + 8
+        except IndexError:
+            return None
+
+    audio = None
+    p = 0
+    audio_size = None
+    audios = []
+
+    for chunk in res.iter_content(chunk_size=1024):
+        if audio is None:
+            audio = chunk
+        else:
+            audio += chunk
+
+        p += len(chunk)
+        if audio_size is not None:
+            if p >= audio_size:
+                p = p - audio_size
+                audios.append(audio[:audio_size])
+                audio = audio[audio_size:]
+                audio_size = get_file_size_from_bytes(audio)
+        else:
+            audio_size = get_file_size_from_bytes(audio)
+    for i, audio in enumerate(audios):
+        with open(f"{path[:-4]}-{i}.wav", "wb") as f:
+            f.write(audio)
+
+        print(f"{path[:-4]}-{i}.wav")
+    return path
+
+
+# 语音转换 hubert-vits
+def voice_hubert_vits(upload_path, id, format="wav", length=1, noise=0.667, noisew=0.8, save_audio=True,
+                      save_path=None):
+    upload_name = os.path.basename(upload_path)
+    upload_type = f'audio/{upload_name.split(".")[1]}'  # wav,ogg
+
+    with open(upload_path, 'rb') as upload_file:
+        fields = {
+            "upload": (upload_name, upload_file, upload_type),
+            "id": str(id),
+            "format": format,
+            "length": str(length),
+            "noise": str(noise),
+            "noisew": str(noisew),
+        }
+        boundary = '----VoiceConversionFormBoundary' + ''.join(random.sample(string.ascii_letters + string.digits, 16))
+
+        m = MultipartEncoder(fields=fields, boundary=boundary)
+        headers = {"Content-Type": m.content_type}
+        url = f"{base_url}/voice/hubert-vits"
+
+        res = requests.post(url=url, data=m, headers=headers)
+    fname = re.findall("filename=(.+)", res.headers["Content-Disposition"])[0]
+    if save_path is not None:
+        path = os.path.join(save_path, fname)
+    else:
+        path = os.path.join(absolute_path, fname)
+    if save_audio:
+        with open(path, "wb") as f:
+            f.write(res.content)
+        print(path)
+        return path
+    return None
+
+
+# 维度情感模型 w2v2-vits
+def voice_w2v2_vits(text, id=0, format="wav", lang="auto", length=1, noise=0.667, noisew=0.8, max=50, emotion=0,
+                    save_audio=True, save_path=None):
+    fields = {
+        "text": text,
+        "id": str(id),
+        "format": format,
+        "lang": lang,
+        "length": str(length),
+        "noise": str(noise),
+        "noisew": str(noisew),
+        "max": str(max),
+        "emotion": str(emotion)
+    }
+    boundary = '----VoiceConversionFormBoundary' + ''.join(random.sample(string.ascii_letters + string.digits, 16))
+
+    m = MultipartEncoder(fields=fields, boundary=boundary)
+    headers = {"Content-Type": m.content_type}
+    url = f"{base_url}/voice/w2v2-vits"
+
+    res = requests.post(url=url, data=m, headers=headers)
+    fname = re.findall("filename=(.+)", res.headers["Content-Disposition"])[0]
+    if save_path is not None:
+        path = os.path.join(save_path, fname)
+    else:
+        path = os.path.join(absolute_path, fname)
+    if save_audio:
+        with open(path, "wb") as f:
+            f.write(res.content)
+        print(path)
+        return path
+    return None
+
+
+# 语音转换 同VITS模型内角色之间的音色转换
+def voice_conversion(upload_path, original_id, target_id, save_audio=True, save_path=None):
+    upload_name = os.path.basename(upload_path)
+    upload_type = f'audio/{upload_name.split(".")[1]}'  # wav,ogg
+
+    with open(upload_path, 'rb') as upload_file:
+        fields = {
+            "upload": (upload_name, upload_file, upload_type),
+            "original_id": str(original_id),
+            "target_id": str(target_id),
+        }
+        boundary = '----VoiceConversionFormBoundary' + ''.join(random.sample(string.ascii_letters + string.digits, 16))
+        m = MultipartEncoder(fields=fields, boundary=boundary)
+
+        headers = {"Content-Type": m.content_type}
+        url = f"{base_url}/voice/conversion"
+
+        res = requests.post(url=url, data=m, headers=headers)
+
+    fname = re.findall("filename=(.+)", res.headers["Content-Disposition"])[0]
+    if save_path is not None:
+        path = os.path.join(save_path, fname)
+    else:
+        path = os.path.join(absolute_path, fname)
+
+    if save_audio:
+        with open(path, "wb") as f:
+            f.write(res.content)
+        print(path)
+        return path
+    return None
+
+
+def voice_ssml(ssml, save_audio=True, save_path=None):
+    fields = {
+        "ssml": ssml,
+    }
+    boundary = '----VoiceConversionFormBoundary' + ''.join(random.sample(string.ascii_letters + string.digits, 16))
+
+    m = MultipartEncoder(fields=fields, boundary=boundary)
+    headers = {"Content-Type": m.content_type}
+    url = f"{base_url}/voice/ssml"
+
+    res = requests.post(url=url, data=m, headers=headers)
+    fname = re.findall("filename=(.+)", res.headers["Content-Disposition"])[0]
+    if save_path is not None:
+        path = os.path.join(save_path, fname)
+    else:
+        path = os.path.join(absolute_path, fname)
+
+    if save_audio:
+        with open(path, "wb") as f:
+            f.write(res.content)
+        print(path)
+        return path
+    return None
+
+
+def voice_dimensional_emotion(upload_path, save_audio=True,
+                              save_path=None):
+    upload_name = os.path.basename(upload_path)
+    upload_type = f'audio/{upload_name.split(".")[1]}'  # wav,ogg
+
+    with open(upload_path, 'rb') as upload_file:
+        fields = {
+            "upload": (upload_name, upload_file, upload_type),
+        }
+        boundary = '----VoiceConversionFormBoundary' + ''.join(random.sample(string.ascii_letters + string.digits, 16))
+
+        m = MultipartEncoder(fields=fields, boundary=boundary)
+        headers = {"Content-Type": m.content_type}
+        url = f"{base_url}/voice/dimension-emotion"
+
+        res = requests.post(url=url, data=m, headers=headers)
+    fname = re.findall("filename=(.+)", res.headers["Content-Disposition"])[0]
+    if save_path is not None:
+        path = os.path.join(save_path, fname)
+    else:
+        path = os.path.join(absolute_path, fname)
+    if save_audio:
+        with open(path, "wb") as f:
+            f.write(res.content)
+        print(path)
+        return path
+    return None
+
+
+def vits_json(text, id=0, format="wav", lang="auto", length=1, noise=0.667, noisew=0.8, max=50,
+              save_path=None):
+    fields = {
+        "text": text,
+        "id": str(id),
+        "format": format,
+        "lang": lang,
+        "length": str(length),
+        "noise": str(noise),
+        "noisew": str(noisew),
+        "max": str(max)
+    }
+    f = json.dumps(fields)
+    url = f"{base_url}/voice"
+    header = {"Content-Type": 'application/json'}
+    res = requests.post(url=url, data=f, headers=header)
+
+    fname = re.findall("filename=(.+)", res.headers["Content-Disposition"])[0]
+    if save_path is not None:
+        path = os.path.join(save_path, fname)
+    else:
+        path = os.path.join(absolute_path, fname)
+
+    with open(path, "wb") as f:
+        f.write(res.content)
+    print(path)
+    return path
+
+
+# Bert_vits2
+def voice_bert_vits2(text, id=0, format="wav", lang="auto", length=1, noise=0.667, noisew=0.8, max=50, sdp_ratio=0.2,
+                     save_audio=True, save_path=None):
+    fields = {
+        "text": text,
+        "id": str(id),
+        "format": format,
+        "lang": lang,
+        "length": str(length),
+        "noise": str(noise),
+        "noisew": str(noisew),
+        "max": str(max),
+        "sdp_ratio": str(sdp_ratio)
+    }
+    boundary = '----VoiceConversionFormBoundary' + ''.join(random.sample(string.ascii_letters + string.digits, 16))
+
+    m = MultipartEncoder(fields=fields, boundary=boundary)
+    headers = {"Content-Type": m.content_type}
+    url = f"{base_url}/voice/bert-vits2"
+
+    res = requests.post(url=url, data=m, headers=headers)
+    fname = re.findall("filename=(.+)", res.headers["Content-Disposition"])[0]
+    if save_path is not None:
+        path = os.path.join(save_path, fname)
+    else:
+        path = os.path.join(absolute_path, fname)
+    if save_audio:
+        with open(path, "wb") as f:
+            f.write(res.content)
+        print(path)
+        return path
+    return None
+
+
+def test_interface(text):
+    error_num = 0
+    for i in range(100):
+        try:
+            time.sleep(1)
+            t1 = time.time()
+            voice_vits(text, format="wav", lang="zh", save_audio=False)
+            t2 = time.time()
+            print(f"{i}:len:{len(text)}耗时:{t2 - t1}")
+        except Exception as e:
+            error_num += 1
+            print(e)
+    print(f"error_num={error_num}")
+
+
+if __name__ == '__main__':
+    text = "你好,こんにちは"
+
+    ssml = """
+    <speak lang="zh" format="mp3" length="1.2">
+            <voice id="92" >这几天心里颇不宁静。</voice>
+            <voice id="125">今晚在院子里坐着乘凉，忽然想起日日走过的荷塘，在这满月的光里，总该另有一番样子吧。</voice>
+            <voice id="142">月亮渐渐地升高了，墙外马路上孩子们的欢笑，已经听不见了；</voice>
+            <voice id="98">妻在屋里拍着闰儿，迷迷糊糊地哼着眠歌。</voice>
+            <voice id="120">我悄悄地披了大衫，带上门出去。</voice><break time="2s"/>
+            <voice id="121">沿着荷塘，是一条曲折的小煤屑路。</voice>
+            <voice id="122">这是一条幽僻的路；白天也少人走，夜晚更加寂寞。</voice>
+            <voice id="123">荷塘四面，长着许多树，蓊蓊郁郁的。</voice>
+            <voice id="124">路的一旁，是些杨柳，和一些不知道名字的树。</voice>
+            <voice id="125">没有月光的晚上，这路上阴森森的，有些怕人。</voice>
+            <voice id="126">今晚却很好，虽然月光也还是淡淡的。</voice><break time="2s"/>
+            <voice id="127">路上只我一个人，背着手踱着。</voice>
+            <voice id="128">这一片天地好像是我的；我也像超出了平常的自己，到了另一个世界里。</voice>
+            <voice id="129">我爱热闹，也爱冷静；<break strength="x-weak"/>爱群居，也爱独处。</voice>
+            <voice id="130">像今晚上，一个人在这苍茫的月下，什么都可以想，什么都可以不想，便觉是个自由的人。</voice>
+            <voice id="131">白天里一定要做的事，一定要说的话，现在都可不理。</voice>
+            <voice id="132">这是独处的妙处，我且受用这无边的荷香月色好了。</voice>
+        </speak>
+    """
+
+    from config import CACHE_PATH
+
+    path = voice_vits(text, save_path=CACHE_PATH)
+    voice_vits_streaming(text, save_path=CACHE_PATH)
+    voice_w2v2_vits(text, save_path=CACHE_PATH)
+    voice_conversion(path, 1, 3, save_path=CACHE_PATH)
+    voice_hubert_vits(path, 0, save_path=CACHE_PATH)
+    voice_dimensional_emotion(path, save_path=CACHE_PATH)
+    voice_ssml(ssml, save_path=CACHE_PATH)
+    voice_bert_vits2("你好",lang="zh", save_path=CACHE_PATH)
+    voice_bert_vits2("こんにちは", lang="ja", save_path=CACHE_PATH)
+    # os.system(path)

app.py

@@ -6,8 +6,8 @@ from flask import Flask, request, send_file, jsonify, make_response, render_temp
 from werkzeug.utils import secure_filename
 from flask_apscheduler import APScheduler
 from functools import wraps
-from utils.utils import clean_folder, check_is_none
-from utils.merge import merge_model
+from utils.data_utils import save_audio, clean_folder, check_is_none
+from utils.load_model import load_model
 from io import BytesIO
 
 app = Flask(__name__)
@@ -25,7 +25,7 @@ for path in (app.config['LOGS_PATH'], app.config['UPLOAD_FOLDER'], app.config['C
         logger.error(f"Unable to create directory {path}: {str(e)}")
 
 # load model
-tts = merge_model(app.config["MODEL_LIST"])
+tts = load_model(app.config["MODEL_LIST"])
 
 
 def require_api_key(func):
@@ -67,30 +67,23 @@ def voice_speakers_api():
 def voice_vits_api():
     try:
         if request.method == "GET":
-            text = request.args.get("text", "")
-            id = int(request.args.get("id", app.config.get("ID", 0)))
-            format = request.args.get("format", app.config.get("FORMAT", "wav"))
-            lang = request.args.get("lang", app.config.get("LANG", "auto"))
-            length = float(request.args.get("length", app.config.get("LENGTH", 1)))
-            noise = float(request.args.get("noise", app.config.get("NOISE", 0.667)))
-            noisew = float(request.args.get("noisew", app.config.get("NOISEW", 0.8)))
-            max = int(request.args.get("max", app.config.get("MAX", 50)))
-            use_streaming = request.args.get('streaming', False, type=bool)
+            request_data = request.args
         elif request.method == "POST":
             content_type = request.headers.get('Content-Type')
             if content_type == 'application/json':
-                data = request.get_json()
+                request_data = request.get_json()
             else:
-                data = request.form
-            text = data.get("text", "")
-            id = int(data.get("id", app.config.get("ID", 0)))
-            format = data.get("format", app.config.get("FORMAT", "wav"))
-            lang = data.get("lang", app.config.get("LANG", "auto"))
-            length = float(data.get("length", app.config.get("LENGTH", 1)))
-            noise = float(data.get("noise", app.config.get("NOISE", 0.667)))
-            noisew = float(data.get("noisew", app.config.get("NOISEW", 0.8)))
-            max = int(data.get("max", app.config.get("MAX", 50)))
-            use_streaming = request.form.get('streaming', False, type=bool)
+                request_data = request.form
+
+        text = request_data.get("text", "")
+        id = int(request_data.get("id", app.config.get("ID", 0)))
+        format = request_data.get("format", app.config.get("FORMAT", "wav"))
+        lang = request_data.get("lang", app.config.get("LANG", "auto")).lower()
+        length = float(request_data.get("length", app.config.get("LENGTH", 1)))
+        noise = float(request_data.get("noise", app.config.get("NOISE", 0.667)))
+        noisew = float(request_data.get("noisew", app.config.get("NOISEW", 0.8)))
+        max = int(request_data.get("max", app.config.get("MAX", 50)))
+        use_streaming = request_data.get('streaming', False, type=bool)
     except Exception as e:
         logger.error(f"[VITS] {e}")
         return make_response("parameter error", 400)
@@ -112,7 +105,7 @@ def voice_vits_api():
 
     # 校验模型是否支持输入的语言
     speaker_lang = tts.voice_speakers["VITS"][id].get('lang')
-    if lang.upper() != "AUTO" and lang.upper() != "MIX" and len(speaker_lang) != 1 and lang not in speaker_lang:
+    if lang not in ["auto", "mix"] and len(speaker_lang) != 1 and lang not in speaker_lang:
         logger.info(f"[VITS] lang \"{lang}\" is not in {speaker_lang}")
         return make_response(jsonify({"status": "error", "message": f"lang '{lang}' is not in {speaker_lang}"}), 400)
 
@@ -136,20 +129,23 @@ def voice_vits_api():
             "lang": lang,
             "speaker_lang": speaker_lang}
 
-    if app.config.get("SAVE_AUDIO", False):
-        logger.debug(f"[VITS] {fname}")
-
     if use_streaming:
-        audio = tts.stream_vits_infer(task, fname)
+        audio = tts.stream_vits_infer(task)
         response = make_response(audio)
         response.headers['Content-Disposition'] = f'attachment; filename={fname}'
         response.headers['Content-Type'] = file_type
         return response
     else:
         t1 = time.time()
-        audio = tts.vits_infer(task, fname)
+        audio = tts.vits_infer(task)
         t2 = time.time()
         logger.info(f"[VITS] finish in {(t2 - t1):.2f}s")
+
+        if app.config.get("SAVE_AUDIO", False):
+            logger.debug(f"[VITS] {fname}")
+            path = os.path.join(app.config.get('CACHE_PATH'), fname)
+            save_audio(audio.getvalue(), path)
+
         return send_file(path_or_file=audio, mimetype=file_type, download_name=fname)
 
 
@@ -191,11 +187,15 @@ def voice_hubert_api():
             "audio_path": os.path.join(app.config['UPLOAD_FOLDER'], fname)}
 
     t1 = time.time()
-    audio = tts.hubert_vits_infer(task, fname)
+    audio = tts.hubert_vits_infer(task)
     t2 = time.time()
+    logger.info(f"[hubert] finish in {(t2 - t1):.2f}s")
+
     if app.config.get("SAVE_AUDIO", False):
         logger.debug(f"[hubert] {fname}")
-    logger.info(f"[hubert] finish in {(t2 - t1):.2f}s")
+        path = os.path.join(app.config.get('CACHE_PATH'), fname)
+        save_audio(audio.getvalue(), path)
+
     if use_streaming:
         audio = tts.generate_audio_chunks(audio)
         response = make_response(audio)
@@ -211,32 +211,24 @@ def voice_hubert_api():
 def voice_w2v2_api():
     try:
         if request.method == "GET":
-            text = request.args.get("text", "")
-            id = int(request.args.get("id", app.config.get("ID", 0)))
-            format = request.args.get("format", app.config.get("FORMAT", "wav"))
-            lang = request.args.get("lang", app.config.get("LANG", "auto"))
-            length = float(request.args.get("length", app.config.get("LENGTH", 1)))
-            noise = float(request.args.get("noise", app.config.get("NOISE", 0.667)))
-            noisew = float(request.args.get("noisew", app.config.get("NOISEW", 0.8)))
-            max = int(request.args.get("max", app.config.get("MAX", 50)))
-            emotion = int(request.args.get("emotion", app.config.get("EMOTION", 0)))
-            use_streaming = request.args.get('streaming', False, type=bool)
+            request_data = request.args
         elif request.method == "POST":
             content_type = request.headers.get('Content-Type')
             if content_type == 'application/json':
-                data = request.get_json()
+                request_data = request.get_json()
             else:
-                data = request.form
-            text = data.get("text", "")
-            id = int(data.get("id", app.config.get("ID", 0)))
-            format = data.get("format", app.config.get("FORMAT", "wav"))
-            lang = data.get("lang", app.config.get("LANG", "auto"))
-            length = float(data.get("length"))
-            noise = float(data.get("noise", app.config.get("NOISE", 0.667)))
-            noisew = float(data.get("noisew", app.config.get("NOISEW", 0.8)))
-            max = int(data.get("max", app.config.get("MAX", 50)))
-            emotion = int(data.get("emotion", app.config.get("EMOTION", 0)))
-            use_streaming = request.form.get('streaming', False, type=bool)
+                request_data = request.form
+
+        text = request_data.get("text", "")
+        id = int(request_data.get("id", app.config.get("ID", 0)))
+        format = request_data.get("format", app.config.get("FORMAT", "wav"))
+        lang = request_data.get("lang", app.config.get("LANG", "auto")).lower()
+        length = float(request_data.get("length", app.config.get("LENGTH", 1)))
+        noise = float(request_data.get("noise", app.config.get("NOISE", 0.667)))
+        noisew = float(request_data.get("noisew", app.config.get("NOISEW", 0.8)))
+        max = int(request_data.get("max", app.config.get("MAX", 50)))
+        emotion = int(request_data.get("emotion", app.config.get("EMOTION", 0)))
+        use_streaming = request_data.get('streaming', False, type=bool)
     except Exception as e:
         logger.error(f"[w2v2] {e}")
         return make_response(f"parameter error", 400)
@@ -259,7 +251,7 @@ def voice_w2v2_api():
 
     # 校验模型是否支持输入的语言
     speaker_lang = tts.voice_speakers["W2V2-VITS"][id].get('lang')
-    if lang.upper() != "AUTO" and lang.upper() != "MIX" and len(speaker_lang) != 1 and lang not in speaker_lang:
+    if lang not in ["auto", "mix"] and len(speaker_lang) != 1 and lang not in speaker_lang:
         logger.info(f"[w2v2] lang \"{lang}\" is not in {speaker_lang}")
         return make_response(jsonify({"status": "error", "message": f"lang '{lang}' is not in {speaker_lang}"}), 400)
 
@@ -285,10 +277,15 @@ def voice_w2v2_api():
             "speaker_lang": speaker_lang}
 
     t1 = time.time()
-    audio = tts.w2v2_vits_infer(task, fname)
+    audio = tts.w2v2_vits_infer(task)
     t2 = time.time()
+    logger.info(f"[w2v2] finish in {(t2 - t1):.2f}s")
+
     if app.config.get("SAVE_AUDIO", False):
-        logger.debug(f"[W2V2] {fname}")
+        logger.debug(f"[w2v2] {fname}")
+        path = os.path.join(app.config.get('CACHE_PATH'), fname)
+        save_audio(audio.getvalue(), path)
+
     if use_streaming:
         audio = tts.generate_audio_chunks(audio)
         response = make_response(audio)
@@ -296,7 +293,6 @@ def voice_w2v2_api():
         response.headers['Content-Type'] = file_type
         return response
     else:
-        logger.info(f"[w2v2] finish in {(t2 - t1):.2f}s")
         return send_file(path_or_file=audio, mimetype=file_type, download_name=fname)
 
 
@@ -326,11 +322,15 @@ def vits_voice_conversion_api():
                 "format": format}
 
         t1 = time.time()
-        audio = tts.vits_voice_conversion(task, fname)
+        audio = tts.vits_voice_conversion(task)
         t2 = time.time()
+        logger.info(f"[Voice conversion] finish in {(t2 - t1):.2f}s")
+
         if app.config.get("SAVE_AUDIO", False):
             logger.debug(f"[Voice conversion] {fname}")
-        logger.info(f"[Voice conversion] finish in {(t2 - t1):.2f}s")
+            path = os.path.join(app.config.get('CACHE_PATH'), fname)
+            save_audio(audio.getvalue(), path)
+
         if use_streaming:
             audio = tts.generate_audio_chunks(audio)
             response = make_response(audio)
@@ -343,14 +343,15 @@ def vits_voice_conversion_api():
 
 @app.route('/voice/ssml', methods=["POST"])
 @require_api_key
-def ssml():
+def ssml_api():
     try:
         content_type = request.headers.get('Content-Type')
         if content_type == 'application/json':
-            data = request.get_json()
+            request_data = request.get_json()
         else:
-            data = request.form
-        ssml = data.get("ssml")
+            request_data = request.form
+
+        ssml = request_data.get("ssml")
     except Exception as e:
         logger.info(f"[ssml] {e}")
         return make_response(jsonify({"status": "error", "message": f"parameter error"}), 400)
@@ -361,11 +362,14 @@ def ssml():
     file_type = f"audio/{format}"
 
     t1 = time.time()
-    audio = tts.create_ssml_infer_task(voice_tasks, format, fname)
+    audio = tts.create_ssml_infer_task(voice_tasks, format)
     t2 = time.time()
+    logger.info(f"[ssml] finish in {(t2 - t1):.2f}s")
+
     if app.config.get("SAVE_AUDIO", False):
         logger.debug(f"[ssml] {fname}")
-    logger.info(f"[ssml] finish in {(t2 - t1):.2f}s")
+        path = os.path.join(app.config.get('CACHE_PATH'), fname)
+        save_audio(audio.getvalue(), path)
 
     return send_file(path_or_file=audio, mimetype=file_type, download_name=fname)
 
@@ -385,15 +389,15 @@ def dimensional_emotion():
 
     file_type = "application/octet-stream; charset=ascii"
     fname = os.path.splitext(audio.filename)[0] + ".npy"
-    audio = tts.get_dimensional_emotion_npy(content)
+    emotion_npy = tts.get_dimensional_emotion_npy(content)
     if use_streaming:
-        audio = tts.generate_audio_chunks(audio)
-        response = make_response(audio)
+        emotion_npy = tts.generate_audio_chunks(emotion_npy)
+        response = make_response(emotion_npy)
         response.headers['Content-Disposition'] = f'attachment; filename={fname}'
         response.headers['Content-Type'] = file_type
         return response
     else:
-        return send_file(path_or_file=audio, mimetype=file_type, download_name=fname)
+        return send_file(path_or_file=emotion_npy, mimetype=file_type, download_name=fname)
 
 
 @app.route('/voice/bert-vits2', methods=["GET", "POST"])
@@ -401,37 +405,29 @@ def dimensional_emotion():
 def voice_bert_vits2_api():
     try:
         if request.method == "GET":
-            text = request.args.get("text", "")
-            id = int(request.args.get("id", app.config.get("ID", 0)))
-            format = request.args.get("format", app.config.get("FORMAT", "wav"))
-            # lang = request.args.get("lang", app.config.get("LANG", "auto"))
-            lang = "ZH"
-            length = float(request.args.get("length", app.config.get("LENGTH", 1)))
-            noise = float(request.args.get("noise", app.config.get("NOISE", 0.5)))
-            noisew = float(request.args.get("noisew", app.config.get("NOISEW", 0.6)))
-            sdp_ratio = float(request.args.get("sdp_ratio", 0.2))
-            max = int(request.args.get("max", app.config.get("MAX", 50)))
+            request_data = request.args
         elif request.method == "POST":
             content_type = request.headers.get('Content-Type')
             if content_type == 'application/json':
-                data = request.get_json()
+                request_data = request.get_json()
             else:
-                data = request.form
-            text = data.get("text", "")
-            id = int(data.get("id", app.config.get("ID", 0)))
-            format = data.get("format", app.config.get("FORMAT", "wav"))
-            # lang = data.get("lang", app.config.get("LANG", "auto"))
-            lang = "ZH"
-            length = float(data.get("length", app.config.get("LENGTH", 1)))
-            noise = float(data.get("noise", app.config.get("NOISE", 0.667)))
-            noisew = float(data.get("noisew", app.config.get("NOISEW", 0.8)))
-            sdp_ratio = float(data.get("noisew", app.config.get("SDP_RATIO", 0.2)))
-            max = int(data.get("max", app.config.get("MAX", 50)))
+                request_data = request.form
+
+        text = request_data.get("text", "")
+        id = int(request_data.get("id", app.config.get("ID", 0)))
+        format = request_data.get("format", app.config.get("FORMAT", "wav"))
+        lang = request_data.get("lang", "auto").lower()
+        length = float(request_data.get("length", app.config.get("LENGTH", 1)))
+        noise = float(request_data.get("noise", app.config.get("NOISE", 0.667)))
+        noisew = float(request_data.get("noisew", app.config.get("NOISEW", 0.8)))
+        sdp_ratio = float(request_data.get("sdp_ratio", app.config.get("SDP_RATIO", 0.2)))
+        max = int(request_data.get("max", app.config.get("MAX", 50)))
     except Exception as e:
         logger.error(f"[Bert-VITS2] {e}")
         return make_response("parameter error", 400)
 
-    logger.info(f"[Bert-VITS2] id:{id} format:{format} lang:{lang} length:{length} noise:{noise} noisew:{noisew} sdp_ratio:{sdp_ratio}")
+    logger.info(
+        f"[Bert-VITS2] id:{id} format:{format} lang:{lang} length:{length} noise:{noise} noisew:{noisew} sdp_ratio:{sdp_ratio}")
     logger.info(f"[Bert-VITS2] len:{len(text)} text：{text}")
 
     if check_is_none(text):
@@ -448,7 +444,7 @@ def voice_bert_vits2_api():
 
     # 校验模型是否支持输入的语言
     speaker_lang = tts.voice_speakers["BERT-VITS2"][id].get('lang')
-    if lang.upper() != "AUTO" and lang.upper() != "MIX" and len(speaker_lang) != 1 and lang not in speaker_lang:
+    if lang not in ["auto", "mix"] and len(speaker_lang) != 1 and lang not in speaker_lang:
         logger.info(f"[Bert-VITS2] lang \"{lang}\" is not in {speaker_lang}")
         return make_response(jsonify({"status": "error", "message": f"lang '{lang}' is not in {speaker_lang}"}), 400)
 
@@ -469,14 +465,16 @@ def voice_bert_vits2_api():
             "lang": lang,
             "speaker_lang": speaker_lang}
 
-    if app.config.get("SAVE_AUDIO", False):
-        logger.debug(f"[Bert-VITS2] {fname}")
-
-
     t1 = time.time()
-    audio = tts.bert_vits2_infer(task, fname)
+    audio = tts.bert_vits2_infer(task)
     t2 = time.time()
     logger.info(f"[Bert-VITS2] finish in {(t2 - t1):.2f}s")
+
+    if app.config.get("SAVE_AUDIO", False):
+        logger.debug(f"[Bert-VITS2] {fname}")
+        path = os.path.join(app.config.get('CACHE_PATH'), fname)
+        save_audio(audio.getvalue(), path)
+
     return send_file(path_or_file=audio, mimetype=file_type, download_name=fname)
 
 
@@ -484,16 +482,16 @@ def voice_bert_vits2_api():
 def check():
     try:
         if request.method == "GET":
-            model = request.args.get("model")
-            id = int(request.args.get("id"))
+            request_data = request.args
         elif request.method == "POST":
             content_type = request.headers.get('Content-Type')
             if content_type == 'application/json':
-                data = request.get_json()
+                request_data = request.get_json()
             else:
-                data = request.form
-            model = data.get("model")
-            id = int(data.get("id"))
+                request_data = request.form
+
+        model = request_data.get("model")
+        id = int(request_data.get("id"))
     except Exception as e:
         logger.info(f"[check] {e}")
         return make_response(jsonify({"status": "error", "message": "parameter error"}), 400)

bert_vits2/attentions.py

@@ -42,11 +42,11 @@ class Encoder(nn.Module):
         self.kernel_size = kernel_size
         self.p_dropout = p_dropout
         self.window_size = window_size
-        if isflow:
-            cond_layer = torch.nn.Conv1d(256, 2 * hidden_channels * n_layers, 1)
-            self.cond_pre = torch.nn.Conv1d(hidden_channels, 2 * hidden_channels, 1)
-            self.cond_layer = weight_norm(cond_layer, name='weight')
-            self.gin_channels = 256
+        # if isflow:
+        #     cond_layer = torch.nn.Conv1d(256, 2 * hidden_channels * n_layers, 1)
+        #     self.cond_pre = torch.nn.Conv1d(hidden_channels, 2 * hidden_channels, 1)
+        #     self.cond_layer = weight_norm(cond_layer, name='weight')
+        #     self.gin_channels = 256
         self.cond_layer_idx = self.n_layers
         if 'gin_channels' in kwargs:
             self.gin_channels = kwargs['gin_channels']

bert_vits2/bert/bert-base-japanese-v3/README.md

@@ -0,0 +1,53 @@
+---
+license: apache-2.0
+datasets:
+- cc100
+- wikipedia
+language:
+- ja
+widget:
+- text: 東北大学で[MASK]の研究をしています。
+---
+
+# BERT base Japanese (unidic-lite with whole word masking, CC-100 and jawiki-20230102)
+
+This is a [BERT](https://github.com/google-research/bert) model pretrained on texts in the Japanese language.
+
+This version of the model processes input texts with word-level tokenization based on the Unidic 2.1.2 dictionary (available in [unidic-lite](https://pypi.org/project/unidic-lite/) package), followed by the WordPiece subword tokenization.
+Additionally, the model is trained with the whole word masking enabled for the masked language modeling (MLM) objective.
+
+The codes for the pretraining are available at [cl-tohoku/bert-japanese](https://github.com/cl-tohoku/bert-japanese/).
+
+## Model architecture
+
+The model architecture is the same as the original BERT base model; 12 layers, 768 dimensions of hidden states, and 12 attention heads.
+
+## Training Data
+
+The model is trained on the Japanese portion of [CC-100 dataset](https://data.statmt.org/cc-100/) and the Japanese version of Wikipedia.
+For Wikipedia, we generated a text corpus from the [Wikipedia Cirrussearch dump file](https://dumps.wikimedia.org/other/cirrussearch/) as of January 2, 2023.
+The corpus files generated from CC-100 and Wikipedia are 74.3GB and 4.9GB in size and consist of approximately 392M and 34M sentences, respectively.
+
+For the purpose of splitting texts into sentences, we used [fugashi](https://github.com/polm/fugashi) with [mecab-ipadic-NEologd](https://github.com/neologd/mecab-ipadic-neologd) dictionary (v0.0.7).
+
+## Tokenization
+
+The texts are first tokenized by MeCab with the Unidic 2.1.2 dictionary and then split into subwords by the WordPiece algorithm.
+The vocabulary size is 32768.
+
+We used [fugashi](https://github.com/polm/fugashi) and [unidic-lite](https://github.com/polm/unidic-lite) packages for the tokenization.
+
+## Training
+
+We trained the model first on the CC-100 corpus for 1M steps and then on the Wikipedia corpus for another 1M steps.
+For training of the MLM (masked language modeling) objective, we introduced whole word masking in which all of the subword tokens corresponding to a single word (tokenized by MeCab) are masked at once.
+
+For training of each model, we used a v3-8 instance of Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/).
+
+## Licenses
+
+The pretrained models are distributed under the Apache License 2.0.
+
+## Acknowledgments
+
+This model is trained with Cloud TPUs provided by [TPU Research Cloud](https://sites.research.google/trc/about/) program.

bert_vits2/bert/bert-base-japanese-v3/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "BertForPreTraining"
+    ],
+    "attention_probs_dropout_prob": 0.1,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 768,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_norm_eps": 1e-12,
+    "max_position_embeddings": 512,
+    "model_type": "bert",
+    "num_attention_heads": 12,
+    "num_hidden_layers": 12,
+    "pad_token_id": 0,
+    "type_vocab_size": 2,
+    "vocab_size": 32768
+}

bert_vits2/bert_vits2.py

@@ -1,41 +1,46 @@
-import re
-
 import numpy as np
 import torch
 
-from bert_vits2 import utils, commons
+from bert_vits2 import commons
+from bert_vits2 import utils as bert_vits2_utils
 from bert_vits2.models import SynthesizerTrn
-from bert_vits2.text import symbols, cleaned_text_to_sequence, get_bert
+from bert_vits2.text import *
 from bert_vits2.text.cleaner import clean_text
-from utils.nlp import sentence_split, cut
+from utils import classify_language, get_hparams_from_file, lang_dict
+from utils.sentence import sentence_split_and_markup, cut
 
 
 class Bert_VITS2:
-    def __init__(self, model, config, device=torch.device("cpu")):
-        self.hps_ms = utils.get_hparams_from_file(config)
+    def __init__(self, model, config, device=torch.device("cpu"), **kwargs):
+        self.hps_ms = get_hparams_from_file(config)
         self.n_speakers = getattr(self.hps_ms.data, 'n_speakers', 0)
         self.speakers = [item[0] for item in
                          sorted(list(getattr(self.hps_ms.data, 'spk2id', {'0': 0}).items()), key=lambda x: x[1])]
+
+        self.legacy = getattr(self.hps_ms.data, 'legacy', False)
+        self.symbols = symbols_legacy if self.legacy else symbols
+        self._symbol_to_id = {s: i for i, s in enumerate(self.symbols)}
+
         self.net_g = SynthesizerTrn(
-            len(symbols),
+            len(self.symbols),
             self.hps_ms.data.filter_length // 2 + 1,
             self.hps_ms.train.segment_size // self.hps_ms.data.hop_length,
             n_speakers=self.hps_ms.data.n_speakers,
+            symbols=self.symbols,
             **self.hps_ms.model).to(device)
         _ = self.net_g.eval()
         self.device = device
         self.load_model(model)
 
     def load_model(self, model):
-        utils.load_checkpoint(model, self.net_g, None, skip_optimizer=True)
+        bert_vits2_utils.load_checkpoint(model, self.net_g, None, skip_optimizer=True)
 
     def get_speakers(self):
         return self.speakers
 
     def get_text(self, text, language_str, hps):
         norm_text, phone, tone, word2ph = clean_text(text, language_str)
-        # print([f"{p}{t}" for p, t in zip(phone, tone)])
-        phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
+        phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str, self._symbol_to_id)
 
         if hps.data.add_blank:
             phone = commons.intersperse(phone, 0)
@@ -45,25 +50,37 @@ class Bert_VITS2:
                 word2ph[i] = word2ph[i] * 2
             word2ph[0] += 1
         bert = get_bert(norm_text, word2ph, language_str)
+        del word2ph
+        assert bert.shape[-1] == len(phone), phone
 
-        assert bert.shape[-1] == len(phone)
-
+        if language_str == "zh":
+            bert = bert
+            ja_bert = torch.zeros(768, len(phone))
+        elif language_str == "ja":
+            ja_bert = bert
+            bert = torch.zeros(1024, len(phone))
+        else:
+            bert = torch.zeros(1024, len(phone))
+            ja_bert = torch.zeros(768, len(phone))
+        assert bert.shape[-1] == len(
+            phone
+        ), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
         phone = torch.LongTensor(phone)
         tone = torch.LongTensor(tone)
         language = torch.LongTensor(language)
+        return bert, ja_bert, phone, tone, language
 
-        return bert, phone, tone, language
-
-    def infer(self, text, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid):
-        bert, phones, tones, lang_ids = self.get_text(text, "ZH", self.hps_ms)
+    def infer(self, text, lang, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid):
+        bert, ja_bert, phones, tones, lang_ids = self.get_text(text, lang, self.hps_ms)
         with torch.no_grad():
             x_tst = phones.to(self.device).unsqueeze(0)
             tones = tones.to(self.device).unsqueeze(0)
             lang_ids = lang_ids.to(self.device).unsqueeze(0)
             bert = bert.to(self.device).unsqueeze(0)
+            ja_bert = ja_bert.to(self.device).unsqueeze(0)
             x_tst_lengths = torch.LongTensor([phones.size(0)]).to(self.device)
             speakers = torch.LongTensor([int(sid)]).to(self.device)
-            audio = self.net_g.infer(x_tst, x_tst_lengths, speakers, tones, lang_ids, bert, sdp_ratio=sdp_ratio
+            audio = self.net_g.infer(x_tst, x_tst_lengths, speakers, tones, lang_ids, bert, ja_bert, sdp_ratio=sdp_ratio
                                      , noise_scale=noise_scale, noise_scale_w=noise_scale_w, length_scale=length_scale)[
                 0][0, 0].data.cpu().float().numpy()
 
@@ -72,17 +89,20 @@ class Bert_VITS2:
 
     def get_audio(self, voice, auto_break=False):
         text = voice.get("text", None)
+        lang = voice.get("lang", "auto")
         sdp_ratio = voice.get("sdp_ratio", 0.2)
         noise_scale = voice.get("noise", 0.5)
         noise_scale_w = voice.get("noisew", 0.6)
         length_scale = voice.get("length", 1)
         sid = voice.get("id", 0)
         max = voice.get("max", 50)
-        # sentence_list = sentence_split(text, max, "ZH", ["zh"])
+        # sentence_list = sentence_split_and_markup(text, max, "ZH", ["zh"])
+        if lang == "auto":
+            lang = classify_language(text, target_languages=lang_dict["bert_vits2"])
         sentence_list = cut(text, max)
         audios = []
         for sentence in sentence_list:
-            audio = self.infer(sentence, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid)
+            audio = self.infer(sentence, lang, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid)
             audios.append(audio)
         audio = np.concatenate(audios)
         return audio

bert_vits2/models.py

@@ -11,7 +11,7 @@ from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d
 from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
 
 from bert_vits2.commons import init_weights, get_padding
-from bert_vits2.text import symbols, num_tones, num_languages
+from bert_vits2.text import num_tones, num_languages
 
 
 class DurationDiscriminator(nn.Module):  # vits2
@@ -254,7 +254,8 @@ class TextEncoder(nn.Module):
                  n_layers,
                  kernel_size,
                  p_dropout,
-                 gin_channels=0):
+                 gin_channels=0,
+                 symbols=None):
         super().__init__()
         self.n_vocab = n_vocab
         self.out_channels = out_channels
@@ -272,6 +273,7 @@ class TextEncoder(nn.Module):
         self.language_emb = nn.Embedding(num_languages, hidden_channels)
         nn.init.normal_(self.language_emb.weight, 0.0, hidden_channels ** -0.5)
         self.bert_proj = nn.Conv1d(1024, hidden_channels, 1)
+        self.ja_bert_proj = nn.Conv1d(768, hidden_channels, 1)
 
         self.encoder = attentions.Encoder(
             hidden_channels,
@@ -283,9 +285,10 @@ class TextEncoder(nn.Module):
             gin_channels=self.gin_channels)
         self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
 
-    def forward(self, x, x_lengths, tone, language, bert, g=None):
-        x = (self.emb(x) + self.tone_emb(tone) + self.language_emb(language) + self.bert_proj(bert).transpose(1,
-                                                                                                              2)) * math.sqrt(
+    def forward(self, x, x_lengths, tone, language, bert, ja_bert, g=None):
+        bert_emb = self.bert_proj(bert).transpose(1, 2)
+        ja_bert_emb = self.ja_bert_proj(ja_bert).transpose(1, 2)
+        x = (self.emb(x) + self.tone_emb(tone) + self.language_emb(language) + bert_emb + ja_bert_emb) * math.sqrt(
             self.hidden_channels)  # [b, t, h]
         x = torch.transpose(x, 1, -1)  # [b, h, t]
         x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype)
@@ -620,6 +623,7 @@ class SynthesizerTrn(nn.Module):
         self.current_mas_noise_scale = self.mas_noise_scale_initial
         if self.use_spk_conditioned_encoder and gin_channels > 0:
             self.enc_gin_channels = gin_channels
+        symbols = kwargs.get("symbols")
         self.enc_p = TextEncoder(n_vocab,
                                  inter_channels,
                                  hidden_channels,
@@ -628,7 +632,9 @@ class SynthesizerTrn(nn.Module):
                                  n_layers,
                                  kernel_size,
                                  p_dropout,
-                                 gin_channels=self.enc_gin_channels)
+                                 gin_channels=self.enc_gin_channels,
+                                 symbols=symbols,
+                                 )
         self.dec = Generator(inter_channels, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates,
                              upsample_initial_channel, upsample_kernel_sizes, gin_channels=gin_channels)
         self.enc_q = PosteriorEncoder(spec_channels, inter_channels, hidden_channels, 5, 1, 16,
@@ -643,12 +649,13 @@ class SynthesizerTrn(nn.Module):
         self.sdp = StochasticDurationPredictor(hidden_channels, 192, 3, 0.5, 4, gin_channels=gin_channels)
         self.dp = DurationPredictor(hidden_channels, 256, 3, 0.5, gin_channels=gin_channels)
 
-        if n_speakers >= 1:
-            self.emb_g = nn.Embedding(n_speakers, gin_channels)
+        if self.n_speakers > 0:
+            self.emb_g = nn.Embedding(self.n_speakers, gin_channels)
         else:
             self.ref_enc = ReferenceEncoder(spec_channels, gin_channels)
 
-    def infer(self, x, x_lengths, sid, tone, language, bert, noise_scale=.667, length_scale=1, noise_scale_w=0.8,
+    def infer(self, x, x_lengths, sid, tone, language, bert, ja_bert, noise_scale=.667, length_scale=1,
+              noise_scale_w=0.8,
               max_len=None, sdp_ratio=0, y=None):
         # x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths, tone, language, bert)
         # g = self.gst(y)
@@ -656,7 +663,7 @@ class SynthesizerTrn(nn.Module):
             g = self.emb_g(sid).unsqueeze(-1)  # [b, h, 1]
         else:
             g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
-        x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths, tone, language, bert, g=g)
+        x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths, tone, language, bert, ja_bert, g=g)
         logw = self.sdp(x, x_mask, g=g, reverse=True, noise_scale=noise_scale_w) * (sdp_ratio) + self.dp(x, x_mask,
                                                                                                          g=g) * (
                        1 - sdp_ratio)

bert_vits2/text/__init__.py

@@ -1,17 +1,16 @@
 from bert_vits2.text.symbols import *
 from .chinese_bert import get_bert_feature as zh_bert
 from .english_bert_mock import get_bert_feature as en_bert
+from .japanese_bert import get_bert_feature as ja_bert
 
-_symbol_to_id = {s: i for i, s in enumerate(symbols)}
 
-
-def cleaned_text_to_sequence(cleaned_text, tones, language):
-    '''Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
-      Args:
-        text: string to convert to a sequence
-      Returns:
-        List of integers corresponding to the symbols in the text
-    '''
+def cleaned_text_to_sequence(cleaned_text, tones, language, _symbol_to_id):
+    """Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
+    Args:
+      text: string to convert to a sequence
+    Returns:
+      List of integers corresponding to the symbols in the text
+    """
     phones = [_symbol_to_id[symbol] for symbol in cleaned_text]
     tone_start = language_tone_start_map[language]
     tones = [i + tone_start for i in tones]
@@ -21,9 +20,6 @@ def cleaned_text_to_sequence(cleaned_text, tones, language):
 
 
 def get_bert(norm_text, word2ph, language):
-    lang_bert_func_map = {
-        'ZH': zh_bert,
-        'EN': en_bert
-    }
+    lang_bert_func_map = {"zh": zh_bert, "en": en_bert, "ja": ja_bert}
     bert = lang_bert_func_map[language](norm_text, word2ph)
     return bert

bert_vits2/text/chinese.py

@@ -12,6 +12,8 @@ pinyin_to_symbol_map = {line.split("\t")[0]: line.strip().split("\t")[1] for lin
                         open(os.path.join(current_file_path, 'opencpop-strict.txt')).readlines()}
 
 import jieba.posseg as psg
+from jieba import lcut
+lcut("预加载")
 
 rep_map = {
     '：': ',',

bert_vits2/text/chinese_bert.py

@@ -1,9 +1,20 @@
+import os
+
 import config
 import torch
 from transformers import AutoTokenizer, AutoModelForMaskedLM
 from logger import logger
+from utils.download import download_and_verify
+from config import DEVICE as device
+
+URLS = [
+    "https://huggingface.co/hfl/chinese-roberta-wwm-ext-large/resolve/main/pytorch_model.bin",
+]
+TARGET_PATH = os.path.join(config.ABS_PATH, "bert_vits2/bert/chinese-roberta-wwm-ext-large/pytorch_model.bin")
+EXPECTED_MD5 = None
 
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+if not os.path.exists(TARGET_PATH):
+    success, message = download_and_verify(URLS, TARGET_PATH, EXPECTED_MD5)
 
 try:
     logger.info("Loading chinese-roberta-wwm-ext-large...")
@@ -13,10 +24,10 @@ try:
     logger.info("Loading finished.")
 except Exception as e:
     logger.error(e)
-    logger.error(f"Please download model from hfl/chinese-roberta-wwm-ext-large.")
+    logger.error(f"Please download pytorch_model.bin from hfl/chinese-roberta-wwm-ext-large.")
 
 
-def get_bert_feature(text, word2ph):
+def get_bert_feature(text, word2ph, device=config.DEVICE):
     with torch.no_grad():
         inputs = tokenizer(text, return_tensors='pt')
         for i in inputs:
@@ -37,7 +48,6 @@ def get_bert_feature(text, word2ph):
 
 
 if __name__ == '__main__':
-    # feature = get_bert_feature('你好,我是说的道理。')
     import torch
 
     word_level_feature = torch.rand(38, 1024)  # 12个词,每个词1024维特征

bert_vits2/text/cleaner.py

@@ -1,7 +1,8 @@
-from bert_vits2.text import chinese, cleaned_text_to_sequence
+from bert_vits2.text import chinese, japanese, cleaned_text_to_sequence
 
 language_module_map = {
-    'ZH': chinese
+    'zh': chinese,
+    'ja': japanese
 }
 
 

bert_vits2/text/japanese.py

@@ -1,104 +1,585 @@
-# modified from https://github.com/CjangCjengh/vits/blob/main/text/japanese.py
+# Convert Japanese text to phonemes which is
+# compatible with Julius https://github.com/julius-speech/segmentation-kit
 import re
-import sys
-
-import pyopenjtalk
-
-from bert_vits2.text import symbols
-
-# Regular expression matching Japanese without punctuation marks:
-_japanese_characters = re.compile(
-    r'[A-Za-z\d\u3005\u3040-\u30ff\u4e00-\u9fff\uff11-\uff19\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d]')
-
-# Regular expression matching non-Japanese characters or punctuation marks:
-_japanese_marks = re.compile(
-    r'[^A-Za-z\d\u3005\u3040-\u30ff\u4e00-\u9fff\uff11-\uff19\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d]')
-
-# List of (symbol, Japanese) pairs for marks:
-_symbols_to_japanese = [(re.compile('%s' % x[0]), x[1]) for x in [
-    ('％', 'パーセント')
-]]
-
-# List of (consonant, sokuon) pairs:
-_real_sokuon = [(re.compile('%s' % x[0]), x[1]) for x in [
-    (r'Q([↑↓]*[kg])', r'k#\1'),
-    (r'Q([↑↓]*[tdjʧ])', r't#\1'),
-    (r'Q([↑↓]*[sʃ])', r's\1'),
-    (r'Q([↑↓]*[pb])', r'p#\1')
-]]
-
-# List of (consonant, hatsuon) pairs:
-_real_hatsuon = [(re.compile('%s' % x[0]), x[1]) for x in [
-    (r'N([↑↓]*[pbm])', r'm\1'),
-    (r'N([↑↓]*[ʧʥj])', r'n^\1'),
-    (r'N([↑↓]*[tdn])', r'n\1'),
-    (r'N([↑↓]*[kg])', r'ŋ\1')
-]]
-
-
-def post_replace_ph(ph):
-    rep_map = {
-        '：': ',',
-        '；': ',',
-        '，': ',',
-        '。': '.',
-        '！': '!',
-        '？': '?',
-        '\n': '.',
-        "·": ",",
-        '、': ",",
-        '...': '…',
-        'v': "V"
-    }
-    if ph in rep_map.keys():
-        ph = rep_map[ph]
-    if ph in symbols:
-        return ph
-    if ph not in symbols:
-        ph = 'UNK'
-    return ph
-
-
-def symbols_to_japanese(text):
-    for regex, replacement in _symbols_to_japanese:
-        text = re.sub(regex, replacement, text)
-    return text
-
-
-def preprocess_jap(text):
-    '''Reference https://r9y9.github.io/ttslearn/latest/notebooks/ch10_Recipe-Tacotron.html'''
-    text = symbols_to_japanese(text)
-    sentences = re.split(_japanese_marks, text)
-    marks = re.findall(_japanese_marks, text)
-    text = []
-    for i, sentence in enumerate(sentences):
-        if re.match(_japanese_characters, sentence):
-            p = pyopenjtalk.g2p(sentence)
-            text += p.split(" ")
-
-        if i < len(marks):
-            text += [marks[i].replace(' ', '')]
-    return text
+import unicodedata
+
+from transformers import AutoTokenizer
+
+from bert_vits2.text.symbols import *
+from bert_vits2.text.japanese_bert import tokenizer
+
+try:
+    import MeCab
+except ImportError as e:
+    raise ImportError("Japanese requires mecab-python3 and unidic-lite.") from e
+from num2words import num2words
+
+_CONVRULES = [
+    # Conversion of 2 letters
+    "アァ/ a a",
+    "イィ/ i i",
+    "イェ/ i e",
+    "イャ/ y a",
+    "ウゥ/ u:",
+    "エェ/ e e",
+    "オォ/ o:",
+    "カァ/ k a:",
+    "キィ/ k i:",
+    "クゥ/ k u:",
+    "クャ/ ky a",
+    "クュ/ ky u",
+    "クョ/ ky o",
+    "ケェ/ k e:",
+    "コォ/ k o:",
+    "ガァ/ g a:",
+    "ギィ/ g i:",
+    "グゥ/ g u:",
+    "グャ/ gy a",
+    "グュ/ gy u",
+    "グョ/ gy o",
+    "ゲェ/ g e:",
+    "ゴォ/ g o:",
+    "サァ/ s a:",
+    "シィ/ sh i:",
+    "スゥ/ s u:",
+    "スャ/ sh a",
+    "スュ/ sh u",
+    "スョ/ sh o",
+    "セェ/ s e:",
+    "ソォ/ s o:",
+    "ザァ/ z a:",
+    "ジィ/ j i:",
+    "ズゥ/ z u:",
+    "ズャ/ zy a",
+    "ズュ/ zy u",
+    "ズョ/ zy o",
+    "ゼェ/ z e:",
+    "ゾォ/ z o:",
+    "タァ/ t a:",
+    "チィ/ ch i:",
+    "ツァ/ ts a",
+    "ツィ/ ts i",
+    "ツゥ/ ts u:",
+    "ツャ/ ch a",
+    "ツュ/ ch u",
+    "ツョ/ ch o",
+    "ツェ/ ts e",
+    "ツォ/ ts o",
+    "テェ/ t e:",
+    "トォ/ t o:",
+    "ダァ/ d a:",
+    "ヂィ/ j i:",
+    "ヅゥ/ d u:",
+    "ヅャ/ zy a",
+    "ヅュ/ zy u",
+    "ヅョ/ zy o",
+    "デェ/ d e:",
+    "ドォ/ d o:",
+    "ナァ/ n a:",
+    "ニィ/ n i:",
+    "ヌゥ/ n u:",
+    "ヌャ/ ny a",
+    "ヌュ/ ny u",
+    "ヌョ/ ny o",
+    "ネェ/ n e:",
+    "ノォ/ n o:",
+    "ハァ/ h a:",
+    "ヒィ/ h i:",
+    "フゥ/ f u:",
+    "フャ/ hy a",
+    "フュ/ hy u",
+    "フョ/ hy o",
+    "ヘェ/ h e:",
+    "ホォ/ h o:",
+    "バァ/ b a:",
+    "ビィ/ b i:",
+    "ブゥ/ b u:",
+    "フャ/ hy a",
+    "ブュ/ by u",
+    "フョ/ hy o",
+    "ベェ/ b e:",
+    "ボォ/ b o:",
+    "パァ/ p a:",
+    "ピィ/ p i:",
+    "プゥ/ p u:",
+    "プャ/ py a",
+    "プュ/ py u",
+    "プョ/ py o",
+    "ペェ/ p e:",
+    "ポォ/ p o:",
+    "マァ/ m a:",
+    "ミィ/ m i:",
+    "ムゥ/ m u:",
+    "ムャ/ my a",
+    "ムュ/ my u",
+    "ムョ/ my o",
+    "メェ/ m e:",
+    "モォ/ m o:",
+    "ヤァ/ y a:",
+    "ユゥ/ y u:",
+    "ユャ/ y a:",
+    "ユュ/ y u:",
+    "ユョ/ y o:",
+    "ヨォ/ y o:",
+    "ラ���/ r a:",
+    "リィ/ r i:",
+    "ルゥ/ r u:",
+    "ルャ/ ry a",
+    "ルュ/ ry u",
+    "ルョ/ ry o",
+    "レェ/ r e:",
+    "ロォ/ r o:",
+    "ワァ/ w a:",
+    "ヲォ/ o:",
+    "ディ/ d i",
+    "デェ/ d e:",
+    "デャ/ dy a",
+    "デュ/ dy u",
+    "デョ/ dy o",
+    "ティ/ t i",
+    "テェ/ t e:",
+    "テャ/ ty a",
+    "テュ/ ty u",
+    "テョ/ ty o",
+    "スィ/ s i",
+    "ズァ/ z u a",
+    "ズィ/ z i",
+    "ズゥ/ z u",
+    "ズャ/ zy a",
+    "ズュ/ zy u",
+    "ズョ/ zy o",
+    "ズェ/ z e",
+    "ズォ/ z o",
+    "キャ/ ky a",
+    "キュ/ ky u",
+    "キョ/ ky o",
+    "シャ/ sh a",
+    "シュ/ sh u",
+    "シェ/ sh e",
+    "ショ/ sh o",
+    "チャ/ ch a",
+    "チュ/ ch u",
+    "チェ/ ch e",
+    "チョ/ ch o",
+    "トゥ/ t u",
+    "トャ/ ty a",
+    "トュ/ ty u",
+    "トョ/ ty o",
+    "ドァ/ d o a",
+    "ドゥ/ d u",
+    "ドャ/ dy a",
+    "ドュ/ dy u",
+    "ドョ/ dy o",
+    "ドォ/ d o:",
+    "ニャ/ ny a",
+    "ニュ/ ny u",
+    "ニョ/ ny o",
+    "ヒャ/ hy a",
+    "ヒュ/ hy u",
+    "ヒョ/ hy o",
+    "ミャ/ my a",
+    "ミュ/ my u",
+    "ミョ/ my o",
+    "リャ/ ry a",
+    "リュ/ ry u",
+    "リョ/ ry o",
+    "ギャ/ gy a",
+    "ギュ/ gy u",
+    "ギョ/ gy o",
+    "ヂェ/ j e",
+    "ヂャ/ j a",
+    "ヂュ/ j u",
+    "ヂョ/ j o",
+    "ジェ/ j e",
+    "ジャ/ j a",
+    "ジュ/ j u",
+    "ジョ/ j o",
+    "ビャ/ by a",
+    "ビュ/ by u",
+    "ビョ/ by o",
+    "ピャ/ py a",
+    "ピュ/ py u",
+    "ピョ/ py o",
+    "ウァ/ u a",
+    "ウィ/ w i",
+    "ウェ/ w e",
+    "ウォ/ w o",
+    "ファ/ f a",
+    "フィ/ f i",
+    "フゥ/ f u",
+    "フャ/ hy a",
+    "フュ/ hy u",
+    "フョ/ hy o",
+    "フェ/ f e",
+    "フォ/ f o",
+    "ヴァ/ b a",
+    "ヴィ/ b i",
+    "ヴェ/ b e",
+    "ヴォ/ b o",
+    "ヴュ/ by u",
+    # Conversion of 1 letter
+    "ア/ a",
+    "イ/ i",
+    "ウ/ u",
+    "エ/ e",
+    "オ/ o",
+    "カ/ k a",
+    "キ/ k i",
+    "ク/ k u",
+    "ケ/ k e",
+    "コ/ k o",
+    "サ/ s a",
+    "シ/ sh i",
+    "ス/ s u",
+    "セ/ s e",
+    "ソ/ s o",
+    "タ/ t a",
+    "チ/ ch i",
+    "ツ/ ts u",
+    "テ/ t e",
+    "ト/ t o",
+    "ナ/ n a",
+    "ニ/ n i",
+    "ヌ/ n u",
+    "ネ/ n e",
+    "ノ/ n o",
+    "ハ/ h a",
+    "ヒ/ h i",
+    "フ/ f u",
+    "ヘ/ h e",
+    "ホ/ h o",
+    "マ/ m a",
+    "ミ/ m i",
+    "ム/ m u",
+    "メ/ m e",
+    "モ/ m o",
+    "ラ/ r a",
+    "リ/ r i",
+    "ル/ r u",
+    "レ/ r e",
+    "ロ/ r o",
+    "ガ/ g a",
+    "ギ/ g i",
+    "グ/ g u",
+    "ゲ/ g e",
+    "ゴ/ g o",
+    "ザ/ z a",
+    "ジ/ j i",
+    "ズ/ z u",
+    "ゼ/ z e",
+    "ゾ/ z o",
+    "ダ/ d a",
+    "ヂ/ j i",
+    "ヅ/ z u",
+    "デ/ d e",
+    "ド/ d o",
+    "バ/ b a",
+    "ビ/ b i",
+    "ブ/ b u",
+    "ベ/ b e",
+    "ボ/ b o",
+    "パ/ p a",
+    "ピ/ p i",
+    "プ/ p u",
+    "ペ/ p e",
+    "ポ/ p o",
+    "ヤ/ y a",
+    "ユ/ y u",
+    "ヨ/ y o",
+    "ワ/ w a",
+    "ヰ/ i",
+    "ヱ/ e",
+    "ヲ/ o",
+    "ン/ N",
+    "ッ/ q",
+    "ヴ/ b u",
+    "ー/:",
+    # Try converting broken text
+    "ァ/ a",
+    "ィ/ i",
+    "ゥ/ u",
+    "ェ/ e",
+    "ォ/ o",
+    "ヮ/ w a",
+    "ォ/ o",
+    # Symbols
+    "、/ ,",
+    "。/ .",
+    "！/ !",
+    "？/ ?",
+    "・/ ,",
+]
+
+_COLON_RX = re.compile(":+")
+_REJECT_RX = re.compile("[^ a-zA-Z:,.?]")
+
+
+def _makerulemap():
+    l = [tuple(x.split("/")) for x in _CONVRULES]
+    return tuple({k: v for k, v in l if len(k) == i} for i in (1, 2))
+
+
+_RULEMAP1, _RULEMAP2 = _makerulemap()
+
+
+def kata2phoneme(text: str) -> str:
+    """Convert katakana text to phonemes."""
+    text = text.strip()
+    res = []
+    while text:
+        if len(text) >= 2:
+            x = _RULEMAP2.get(text[:2])
+            if x is not None:
+                text = text[2:]
+                res += x.split(" ")[1:]
+                continue
+        x = _RULEMAP1.get(text[0])
+        if x is not None:
+            text = text[1:]
+            res += x.split(" ")[1:]
+            continue
+        res.append(text[0])
+        text = text[1:]
+    # res = _COLON_RX.sub(":", res)
+    return res
+
+
+_KATAKANA = "".join(chr(ch) for ch in range(ord("ァ"), ord("ン") + 1))
+_HIRAGANA = "".join(chr(ch) for ch in range(ord("ぁ"), ord("ん") + 1))
+_HIRA2KATATRANS = str.maketrans(_HIRAGANA, _KATAKANA)
+
+
+def hira2kata(text: str) -> str:
+    text = text.translate(_HIRA2KATATRANS)
+    return text.replace("う゛", "ヴ")
+
+
+_SYMBOL_TOKENS = set(list("・、。？！"))
+_NO_YOMI_TOKENS = set(list("「」『』―（）［］[]"))
+_TAGGER = MeCab.Tagger()
+
+
+def text2kata(text: str) -> str:
+    parsed = _TAGGER.parse(text)
+    res = []
+    for line in parsed.split("\n"):
+        if line == "EOS":
+            break
+        parts = line.split("\t")
+
+        word, yomi = parts[0], parts[1]
+        if yomi:
+            res.append(yomi)
+        else:
+            if word in _SYMBOL_TOKENS:
+                res.append(word)
+            elif word in ("っ", "ッ"):
+                res.append("ッ")
+            elif word in _NO_YOMI_TOKENS:
+                pass
+            else:
+                res.append(word)
+    return hira2kata("".join(res))
+
+
+_ALPHASYMBOL_YOMI = {
+    "#": "シャープ",
+    "%": "パーセント",
+    "&": "アンド",
+    "+": "プラス",
+    "-": "マイナス",
+    ":": "コロン",
+    ";": "セミコロン",
+    "<": "小なり",
+    "=": "イコール",
+    ">": "大なり",
+    "@": "アット",
+    "a": "エー",
+    "b": "ビー",
+    "c": "シー",
+    "d": "ディー",
+    "e": "イー",
+    "f": "エフ",
+    "g": "ジー",
+    "h": "エイチ",
+    "i": "アイ",
+    "j": "ジェー",
+    "k": "ケー",
+    "l": "エル",
+    "m": "エム",
+    "n": "エヌ",
+    "o": "オー",
+    "p": "ピー",
+    "q": "キュー",
+    "r": "アール",
+    "s": "エス",
+    "t": "ティー",
+    "u": "ユー",
+    "v": "ブイ",
+    "w": "ダブリュー",
+    "x": "エックス",
+    "y": "ワイ",
+    "z": "ゼット",
+    "α": "アルファ",
+    "β": "ベータ",
+    "γ": "ガンマ",
+    "δ": "デルタ",
+    "ε": "イプシロン",
+    "ζ": "ゼータ",
+    "η": "イータ",
+    "θ": "シータ",
+    "ι": "イオタ",
+    "κ": "カッパ",
+    "λ": "ラムダ",
+    "μ": "ミュー",
+    "ν": "ニュー",
+    "ξ": "クサイ",
+    "ο": "オミクロン",
+    "π": "パイ",
+    "ρ": "ロー",
+    "σ": "シグマ",
+    "τ": "タウ",
+    "υ": "ウプシロン",
+    "φ": "ファイ",
+    "χ": "カイ",
+    "ψ": "プサイ",
+    "ω": "オメガ",
+}
+
+_NUMBER_WITH_SEPARATOR_RX = re.compile("[0-9]{1,3}(,[0-9]{3})+")
+_CURRENCY_MAP = {"$": "ドル", "¥": "円", "£": "ポンド", "€": "ユーロ"}
+_CURRENCY_RX = re.compile(r"([$¥£€])([0-9.]*[0-9])")
+_NUMBER_RX = re.compile(r"[0-9]+(\.[0-9]+)?")
+
+
+def japanese_convert_numbers_to_words(text: str) -> str:
+    res = _NUMBER_WITH_SEPARATOR_RX.sub(lambda m: m[0].replace(",", ""), text)
+    res = _CURRENCY_RX.sub(lambda m: m[2] + _CURRENCY_MAP.get(m[1], m[1]), res)
+    res = _NUMBER_RX.sub(lambda m: num2words(m[0], lang="ja"), res)
+    return res
+
+
+def japanese_convert_alpha_symbols_to_words(text: str) -> str:
+    return "".join([_ALPHASYMBOL_YOMI.get(ch, ch) for ch in text.lower()])
+
+
+def japanese_text_to_phonemes(text: str) -> str:
+    """Convert Japanese text to phonemes."""
+    res = unicodedata.normalize("NFKC", text)
+    res = japanese_convert_numbers_to_words(res)
+    # res = japanese_convert_alpha_symbols_to_words(res)
+    res = text2kata(res)
+    res = kata2phoneme(res)
+    return res
+
+
+def is_japanese_character(char):
+    # 定义日语文字系统的 Unicode 范围
+    japanese_ranges = [
+        (0x3040, 0x309F),  # 平假名
+        (0x30A0, 0x30FF),  # 片假名
+        (0x4E00, 0x9FFF),  # 汉字 (CJK Unified Ideographs)
+        (0x3400, 0x4DBF),  # 汉字扩展 A
+        (0x20000, 0x2A6DF),  # 汉字扩展 B
+        # 可以根据需要添加其他汉字扩展范围
+    ]
+
+    # 将字符的 Unicode 编码转换为整数
+    char_code = ord(char)
+
+    # 检查字符是否在任何一个日语范围内
+    for start, end in japanese_ranges:
+        if start <= char_code <= end:
+            return True
+
+    return False
+
+
+rep_map = {
+    "：": ",",
+    "；": ",",
+    "，": ",",
+    "。": ".",
+    "！": "!",
+    "？": "?",
+    "\n": ".",
+    "·": ",",
+    "、": ",",
+    "...": "…",
+}
+
+
+def replace_punctuation(text):
+    pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
+
+    replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
+
+    replaced_text = re.sub(
+        r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF"
+        + "".join(punctuation)
+        + r"]+",
+        "",
+        replaced_text,
+    )
+
+    return replaced_text
 
 
 def text_normalize(text):
-    return text
+    res = unicodedata.normalize("NFKC", text)
+    res = japanese_convert_numbers_to_words(res)
+    # res = "".join([i for i in res if is_japanese_character(i)])
+    res = replace_punctuation(res)
+    return res
+
+
+def distribute_phone(n_phone, n_word):
+    phones_per_word = [0] * n_word
+    for task in range(n_phone):
+        min_tasks = min(phones_per_word)
+        min_index = phones_per_word.index(min_tasks)
+        phones_per_word[min_index] += 1
+    return phones_per_word
 
 
 def g2p(norm_text):
-    phones = preprocess_jap(norm_text)
-    phones = [post_replace_ph(i) for i in phones]
+    tokenized = tokenizer.tokenize(norm_text)
+    phs = []
+    ph_groups = []
+    for t in tokenized:
+        if not t.startswith("#"):
+            ph_groups.append([t])
+        else:
+            ph_groups[-1].append(t.replace("#", ""))
+    word2ph = []
+    for group in ph_groups:
+        phonemes = kata2phoneme(text2kata("".join(group)))
+        # phonemes = [i for i in phonemes if i in symbols]
+        for i in phonemes:
+            assert i in symbols, (i, group, norm_text, tokenized)
+        phone_len = len(phonemes)
+        word_len = len(group)
+
+        aaa = distribute_phone(phone_len, word_len)
+        word2ph += aaa
 
+        phs += phonemes
+    phones = ["_"] + phs + ["_"]
     tones = [0 for i in phones]
-    word2ph = [1 for i in phones]
+    word2ph = [1] + word2ph + [1]
     return phones, tones, word2ph
 
 
-if __name__ == '__main__':
-    for line in open("../../../Downloads/transcript_utf8.txt").readlines():
-        text = line.split(":")[1]
-        phones, tones, word2ph = g2p(text)
-        for p in phones:
-            if p == "z":
-                print(text, phones)
-                sys.exit(0)
+if __name__ == "__main__":
+    from config import ABS_PATH
+
+    tokenizer = AutoTokenizer.from_pretrained(ABS_PATH + "/bert_vits2/bert/bert-base-japanese-v3")
+    text = "hello,こんにちは、世界！……"
+    from bert_vits2.text.japanese_bert import get_bert_feature
+
+    text = text_normalize(text)
+    print(text)
+    phones, tones, word2ph = g2p(text)
+    bert = get_bert_feature(text, word2ph)
+
+    print(phones, tones, word2ph, bert.shape)

bert_vits2/text/japanese_bert.py

@@ -0,0 +1,47 @@
+import os
+
+import torch
+from transformers import AutoTokenizer, AutoModelForMaskedLM
+
+import config
+from logger import logger
+from utils.download import download_and_verify
+from config import DEVICE as device
+
+URLS = [
+    "https://huggingface.co/cl-tohoku/bert-base-japanese-v3/resolve/main/pytorch_model.bin",
+]
+TARGET_PATH = os.path.join(config.ABS_PATH, "bert_vits2/bert/bert-base-japanese-v3/pytorch_model.bin")
+EXPECTED_MD5 = None
+
+if not os.path.exists(TARGET_PATH):
+    success, message = download_and_verify(URLS, TARGET_PATH, EXPECTED_MD5)
+
+try:
+    logger.info("Loading bert-base-japanese-v3...")
+    tokenizer = AutoTokenizer.from_pretrained(config.ABS_PATH + "/bert_vits2/bert/bert-base-japanese-v3")
+    model = AutoModelForMaskedLM.from_pretrained(config.ABS_PATH + "/bert_vits2/bert/bert-base-japanese-v3").to(
+        device)
+    logger.info("Loading finished.")
+except Exception as e:
+    logger.error(e)
+    logger.error(f"Please download pytorch_model.bin from cl-tohoku/bert-base-japanese-v3.")
+
+
+def get_bert_feature(text, word2ph, device=config.DEVICE):
+    with torch.no_grad():
+        inputs = tokenizer(text, return_tensors="pt")
+        for i in inputs:
+            inputs[i] = inputs[i].to(device)
+        res = model(**inputs, output_hidden_states=True)
+        res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
+    assert inputs["input_ids"].shape[-1] == len(word2ph)
+    word2phone = word2ph
+    phone_level_feature = []
+    for i in range(len(word2phone)):
+        repeat_feature = res[i].repeat(word2phone[i], 1)
+        phone_level_feature.append(repeat_feature)
+
+    phone_level_feature = torch.cat(phone_level_feature, dim=0)
+
+    return phone_level_feature.T

bert_vits2/text/symbols.py

@@ -1,52 +1,198 @@
-punctuation = ['!', '?', '…', ",", ".", "'", '-']
+punctuation = ["!", "?", "…", ",", ".", "'", "-"]
 pu_symbols = punctuation + ["SP", "UNK"]
-pad = '_'
+pad = "_"
 
 # chinese
-zh_symbols = ['E', 'En', 'a', 'ai', 'an', 'ang', 'ao', 'b', 'c', 'ch', 'd', 'e', 'ei', 'en', 'eng', 'er', 'f', 'g', 'h',
-              'i', 'i0', 'ia', 'ian', 'iang', 'iao', 'ie', 'in', 'ing', 'iong', 'ir', 'iu', 'j', 'k', 'l', 'm', 'n',
-              'o',
-              'ong',
-              'ou', 'p', 'q', 'r', 's', 'sh', 't', 'u', 'ua', 'uai', 'uan', 'uang', 'ui', 'un', 'uo', 'v', 'van', 've',
-              'vn',
-              'w', 'x', 'y', 'z', 'zh',
-              "AA", "EE", "OO"]
+zh_symbols = [
+    "E",
+    "En",
+    "a",
+    "ai",
+    "an",
+    "ang",
+    "ao",
+    "b",
+    "c",
+    "ch",
+    "d",
+    "e",
+    "ei",
+    "en",
+    "eng",
+    "er",
+    "f",
+    "g",
+    "h",
+    "i",
+    "i0",
+    "ia",
+    "ian",
+    "iang",
+    "iao",
+    "ie",
+    "in",
+    "ing",
+    "iong",
+    "ir",
+    "iu",
+    "j",
+    "k",
+    "l",
+    "m",
+    "n",
+    "o",
+    "ong",
+    "ou",
+    "p",
+    "q",
+    "r",
+    "s",
+    "sh",
+    "t",
+    "u",
+    "ua",
+    "uai",
+    "uan",
+    "uang",
+    "ui",
+    "un",
+    "uo",
+    "v",
+    "van",
+    "ve",
+    "vn",
+    "w",
+    "x",
+    "y",
+    "z",
+    "zh",
+    "AA",
+    "EE",
+    "OO",
+]
 num_zh_tones = 6
 
 # japanese
-ja_symbols = ['I', 'N', 'U', 'a', 'b', 'by', 'ch', 'cl', 'd', 'dy', 'e', 'f', 'g', 'gy', 'h', 'hy', 'i', 'j', 'k', 'ky',
-              'm', 'my', 'n', 'ny', 'o', 'p', 'py', 'r', 'ry', 's', 'sh', 't', 'ts', 'u', 'V', 'w', 'y', 'z']
+ja_symbols_legacy = ['I', 'N', 'U', 'a', 'b', 'by', 'ch', 'cl', 'd', 'dy', 'e', 'f', 'g', 'gy', 'h', 'hy', 'i', 'j',
+                     'k', 'ky',
+                     'm', 'my', 'n', 'ny', 'o', 'p', 'py', 'r', 'ry', 's', 'sh', 't', 'ts', 'u', 'V', 'w', 'y', 'z']
+ja_symbols = [
+    "N",
+    "a",
+    "a:",
+    "b",
+    "by",
+    "ch",
+    "d",
+    "dy",
+    "e",
+    "e:",
+    "f",
+    "g",
+    "gy",
+    "h",
+    "hy",
+    "i",
+    "i:",
+    "j",
+    "k",
+    "ky",
+    "m",
+    "my",
+    "n",
+    "ny",
+    "o",
+    "o:",
+    "p",
+    "py",
+    "q",
+    "r",
+    "ry",
+    "s",
+    "sh",
+    "t",
+    "ts",
+    "ty",
+    "u",
+    "u:",
+    "w",
+    "y",
+    "z",
+    "zy",
+]
 num_ja_tones = 1
 
 # English
-en_symbols = ['aa', 'ae', 'ah', 'ao', 'aw', 'ay', 'b', 'ch', 'd', 'dh', 'eh', 'er', 'ey', 'f', 'g', 'hh', 'ih', 'iy',
-              'jh', 'k', 'l', 'm', 'n', 'ng', 'ow', 'oy', 'p', 'r', 's',
-              'sh', 't', 'th', 'uh', 'uw', 'V', 'w', 'y', 'z', 'zh']
+en_symbols = [
+    "aa",
+    "ae",
+    "ah",
+    "ao",
+    "aw",
+    "ay",
+    "b",
+    "ch",
+    "d",
+    "dh",
+    "eh",
+    "er",
+    "ey",
+    "f",
+    "g",
+    "hh",
+    "ih",
+    "iy",
+    "jh",
+    "k",
+    "l",
+    "m",
+    "n",
+    "ng",
+    "ow",
+    "oy",
+    "p",
+    "r",
+    "s",
+    "sh",
+    "t",
+    "th",
+    "uh",
+    "uw",
+    "V",
+    "w",
+    "y",
+    "z",
+    "zh",
+]
 num_en_tones = 4
 
-# combine all symbols
 normal_symbols = sorted(set(zh_symbols + ja_symbols + en_symbols))
 symbols = [pad] + normal_symbols + pu_symbols
 sil_phonemes_ids = [symbols.index(i) for i in pu_symbols]
 
+# legacy
+normal_symbols_legacy = sorted(set(zh_symbols + ja_symbols_legacy + en_symbols))
+symbols_legacy = [pad] + normal_symbols_legacy + pu_symbols
+sil_phonemes_ids_legacy = [symbols_legacy.index(i) for i in pu_symbols]
+
 # combine all tones
 num_tones = num_zh_tones + num_ja_tones + num_en_tones
 
 # language maps
-language_id_map = {
-    'ZH': 0,
-    "JA": 1,
-    "EN": 2
-}
+language_id_map = {"zh": 0, "ja": 1, "en": 2}
 num_languages = len(language_id_map.keys())
 
 language_tone_start_map = {
-    'ZH': 0,
-    "JA": num_zh_tones,
-    "EN": num_zh_tones + num_ja_tones
+    "zh": 0,
+    "ja": num_zh_tones,
+    "en": num_zh_tones + num_ja_tones,
 }
 
-if __name__ == '__main__':
-    a = set(zh_symbols)
-    b = set(en_symbols)
-    print(sorted(a & b))
+if __name__ == "__main__":
+    zh = set(zh_symbols)
+    en = set(en_symbols)
+    ja = set(ja_symbols)
+    print(zh)
+    print(en)
+    print(ja)
+    print(sorted(zh & en))

bert_vits2/text/tone_sandhi.py

@@ -19,51 +19,442 @@ from pypinyin import lazy_pinyin
 from pypinyin import Style
 
 
-class ToneSandhi():
+class ToneSandhi:
     def __init__(self):
         self.must_neural_tone_words = {
-            '麻烦', '麻利', '鸳鸯', '高粱', '骨头', '骆驼', '马虎', '首饰', '馒头', '馄饨', '风筝',
-            '难为', '队伍', '阔气', '闺女', '门道', '锄头', '铺盖', '铃铛', '铁匠', '钥匙', '里脊',
-            '里头', '部分', '那么', '道士', '造化', '迷糊', '连累', '这么', '这个', '运气', '过去',
-            '软和', '转悠', '踏实', '跳蚤', '跟头', '趔趄', '财主', '豆腐', '讲究', '记性', '记号',
-            '认识', '规矩', '见识', '裁缝', '补丁', '衣裳', '衣服', '衙门', '街坊', '行李', '行当',
-            '蛤蟆', '蘑菇', '薄荷', '葫芦', '葡萄', '萝卜', '荸荠', '苗条', '苗头', '苍蝇', '芝麻',
-            '舒服', '舒坦', '舌头', '自在', '膏药', '脾气', '脑袋', '脊梁', '能耐', '胳膊', '胭脂',
-            '胡萝', '胡琴', '胡同', '聪明', '耽误', '耽搁', '耷拉', '耳朵', '老爷', '老实', '老婆',
-            '老头', '老太', '翻腾', '罗嗦', '罐头', '编辑', '结实', '红火', '累赘', '糨糊', '糊涂',
-            '精神', '粮食', '簸箕', '篱笆', '算计', '算盘', '答应', '笤帚', '笑语', '笑话', '窟窿',
-            '窝囊', '窗户', '稳当', '稀罕', '称呼', '秧歌', '秀气', '秀才', '福气', '祖宗', '砚台',
-            '码头', '石榴', '石头', '石匠', '知识', '眼睛', '眯缝', '眨巴', '眉毛', '相声', '盘算',
-            '白净', '痢疾', '痛快', '疟疾', '疙瘩', '疏忽', '畜生', '生意', '甘蔗', '琵琶', '琢磨',
-            '琉璃', '玻璃', '玫瑰', '玄乎', '狐狸', '状元', '特务', '牲口', '牙碜', '牌楼', '爽快',
-            '爱人', '热闹', '烧饼', '烟筒', '烂糊', '点心', '炊帚', '灯笼', '火候', '漂亮', '滑溜',
-            '溜达', '温和', '清楚', '消息', '浪头', '活泼', '比方', '正经', '欺负', '模糊', '槟榔',
-            '棺材', '棒槌', '棉花', '核桃', '栅栏', '柴火', '架势', '枕头', '枇杷', '机灵', '本事',
-            '木头', '木匠', '朋友', '月饼', '月亮', '暖和', '明白', '时候', '新鲜', '故事', '收拾',
-            '收成', '提防', '挖苦', '挑剔', '指甲', '指头', '拾掇', '拳头', '拨弄', '招牌', '招呼',
-            '抬举', '护士', '折腾', '扫帚', '打量', '打算', '打点', '打扮', '打听', '打发', '扎实',
-            '扁担', '戒指', '懒得', '意识', '意思', '情形', '悟性', '怪物', '思量', '怎么', '念头',
-            '念叨', '快活', '忙活', '志气', '心思', '得罪', '张罗', '弟兄', '开通', '应酬', '庄稼',
-            '干事', '帮手', '帐篷', '希罕', '师父', '师傅', '巴结', '巴掌', '差事', '工夫', '岁数',
-            '屁股', '尾巴', '少爷', '小气', '小伙', '将就', '对头', '对付', '寡妇', '家伙', '客气',
-            '实在', '官司', '学问', '学生', '字号', '嫁妆', '媳妇', '媒人', '婆家', '娘家', '委屈',
-            '姑娘', '姐夫', '妯娌', '妥当', '妖精', '奴才', '女婿', '头发', '太阳', '大爷', '大方',
-            '大意', '大夫', '多少', '多么', '外甥', '壮实', '地道', '地方', '在乎', '困难', '嘴巴',
-            '嘱咐', '嘟囔', '嘀咕', '喜欢', '喇嘛', '喇叭', '商量', '唾沫', '哑巴', '哈欠', '哆嗦',
-            '咳嗽', '和尚', '告诉', '告示', '含糊', '吓唬', '后头', '名字', '名堂', '合同', '吆喝',
-            '叫唤', '口袋', '厚道', '厉害', '千斤', '包袱', '包涵', '匀称', '勤快', '动静', '动弹',
-            '功夫', '力气', '前头', '刺猬', '刺激', '别扭', '利落', '利索', '利害', '分析', '出息',
-            '凑合', '凉快', '冷战', '冤枉', '冒失', '养活', '关系', '先生', '兄弟', '便宜', '使唤',
-            '佩服', '作坊', '体面', '位置', '似的', '伙计', '休息', '什么', '人家', '亲戚', '亲家',
-            '交情', '云彩', '事情', '买卖', '主意', '丫头', '丧气', '两口', '东西', '东家', '世故',
-            '不由', '不在', '下水', '下巴', '上头', '上司', '丈夫', '丈人', '一辈', '那个', '菩萨',
-            '父亲', '母亲', '咕噜', '邋遢', '费用', '冤家', '甜头', '介绍', '荒唐', '大人', '泥鳅',
-            '幸福', '熟悉', '计划', '扑腾', '蜡烛', '姥爷', '照顾', '喉咙', '吉他', '弄堂', '蚂蚱',
-            '凤凰', '拖沓', '寒碜', '糟蹋', '倒腾', '报复', '逻辑', '盘缠', '喽啰', '牢骚', '咖喱',
-            '扫把', '惦记'
+            "麻烦",
+            "麻利",
+            "鸳鸯",
+            "高粱",
+            "骨头",
+            "骆驼",
+            "马虎",
+            "首饰",
+            "馒头",
+            "���饨",
+            "风筝",
+            "难为",
+            "队伍",
+            "阔气",
+            "闺女",
+            "门道",
+            "锄头",
+            "铺盖",
+            "铃铛",
+            "铁匠",
+            "钥匙",
+            "里脊",
+            "里头",
+            "部分",
+            "那么",
+            "道士",
+            "造化",
+            "迷糊",
+            "连累",
+            "这么",
+            "这个",
+            "运气",
+            "过去",
+            "软和",
+            "转悠",
+            "踏实",
+            "跳蚤",
+            "跟头",
+            "趔趄",
+            "财主",
+            "豆腐",
+            "讲究",
+            "记性",
+            "记号",
+            "认识",
+            "规矩",
+            "见识",
+            "裁缝",
+            "补丁",
+            "衣裳",
+            "衣服",
+            "衙门",
+            "街坊",
+            "行李",
+            "行当",
+            "蛤蟆",
+            "蘑菇",
+            "薄荷",
+            "葫芦",
+            "葡萄",
+            "萝卜",
+            "荸荠",
+            "苗条",
+            "苗头",
+            "苍蝇",
+            "芝麻",
+            "舒服",
+            "舒坦",
+            "舌头",
+            "自在",
+            "膏药",
+            "脾气",
+            "脑袋",
+            "脊梁",
+            "能耐",
+            "胳膊",
+            "胭脂",
+            "胡萝",
+            "胡琴",
+            "胡同",
+            "聪明",
+            "耽误",
+            "耽搁",
+            "耷拉",
+            "耳朵",
+            "老爷",
+            "老实",
+            "老婆",
+            "老头",
+            "老太",
+            "翻腾",
+            "罗嗦",
+            "罐头",
+            "编辑",
+            "结实",
+            "红火",
+            "累赘",
+            "糨糊",
+            "糊涂",
+            "精神",
+            "粮食",
+            "簸箕",
+            "篱笆",
+            "算计",
+            "算盘",
+            "答应",
+            "笤帚",
+            "笑语",
+            "笑话",
+            "窟窿",
+            "窝囊",
+            "窗户",
+            "稳当",
+            "稀罕",
+            "称呼",
+            "秧歌",
+            "秀气",
+            "秀才",
+            "福气",
+            "祖宗",
+            "砚台",
+            "码头",
+            "石榴",
+            "石头",
+            "石匠",
+            "知识",
+            "眼睛",
+            "眯缝",
+            "眨巴",
+            "眉毛",
+            "相声",
+            "盘算",
+            "白净",
+            "痢疾",
+            "痛快",
+            "疟疾",
+            "疙瘩",
+            "疏忽",
+            "畜生",
+            "生意",
+            "甘蔗",
+            "琵琶",
+            "琢磨",
+            "琉璃",
+            "玻璃",
+            "玫瑰",
+            "玄乎",
+            "狐狸",
+            "状元",
+            "特务",
+            "牲口",
+            "牙碜",
+            "牌楼",
+            "爽快",
+            "爱人",
+            "热闹",
+            "烧饼",
+            "烟筒",
+            "烂糊",
+            "点心",
+            "炊帚",
+            "灯笼",
+            "火候",
+            "漂亮",
+            "滑溜",
+            "溜达",
+            "温和",
+            "清楚",
+            "消息",
+            "浪头",
+            "活泼",
+            "比方",
+            "正经",
+            "欺负",
+            "模糊",
+            "槟榔",
+            "棺材",
+            "棒槌",
+            "棉花",
+            "核桃",
+            "栅栏",
+            "柴火",
+            "架势",
+            "枕头",
+            "枇杷",
+            "机灵",
+            "本事",
+            "木头",
+            "木匠",
+            "朋友",
+            "月饼",
+            "月亮",
+            "暖和",
+            "明白",
+            "时候",
+            "新鲜",
+            "故事",
+            "收拾",
+            "收成",
+            "提防",
+            "挖苦",
+            "挑剔",
+            "指甲",
+            "指头",
+            "拾掇",
+            "拳头",
+            "拨弄",
+            "招牌",
+            "招呼",
+            "抬举",
+            "护士",
+            "折腾",
+            "扫帚",
+            "打量",
+            "打算",
+            "打点",
+            "打扮",
+            "打听",
+            "打发",
+            "扎实",
+            "扁担",
+            "戒指",
+            "懒得",
+            "意识",
+            "意思",
+            "情形",
+            "悟性",
+            "怪物",
+            "思量",
+            "怎么",
+            "念头",
+            "念叨",
+            "快活",
+            "忙活",
+            "志气",
+            "心思",
+            "得罪",
+            "张罗",
+            "弟兄",
+            "开通",
+            "应酬",
+            "庄稼",
+            "干事",
+            "帮手",
+            "帐篷",
+            "希罕",
+            "师父",
+            "师傅",
+            "巴结",
+            "巴掌",
+            "差事",
+            "工夫",
+            "岁数",
+            "屁股",
+            "尾巴",
+            "少爷",
+            "小气",
+            "小伙",
+            "将就",
+            "对头",
+            "对付",
+            "寡妇",
+            "家伙",
+            "客气",
+            "实在",
+            "官司",
+            "学问",
+            "学生",
+            "字号",
+            "嫁妆",
+            "媳妇",
+            "媒人",
+            "婆家",
+            "娘家",
+            "委屈",
+            "姑娘",
+            "姐夫",
+            "妯娌",
+            "妥当",
+            "妖精",
+            "奴才",
+            "女婿",
+            "头发",
+            "太阳",
+            "大爷",
+            "大方",
+            "大意",
+            "大夫",
+            "多少",
+            "多么",
+            "外甥",
+            "壮实",
+            "地道",
+            "地方",
+            "在乎",
+            "困难",
+            "嘴巴",
+            "嘱咐",
+            "嘟囔",
+            "嘀咕",
+            "喜欢",
+            "喇嘛",
+            "喇叭",
+            "商量",
+            "唾沫",
+            "哑巴",
+            "哈欠",
+            "哆嗦",
+            "咳嗽",
+            "和尚",
+            "告诉",
+            "告示",
+            "含糊",
+            "吓唬",
+            "后头",
+            "名字",
+            "名堂",
+            "合同",
+            "吆喝",
+            "叫唤",
+            "口袋",
+            "厚道",
+            "厉害",
+            "千斤",
+            "包袱",
+            "包涵",
+            "匀称",
+            "勤快",
+            "动静",
+            "动弹",
+            "功夫",
+            "力气",
+            "前头",
+            "刺猬",
+            "刺激",
+            "别扭",
+            "利落",
+            "利索",
+            "利害",
+            "分析",
+            "出息",
+            "凑合",
+            "凉快",
+            "冷战",
+            "冤枉",
+            "冒失",
+            "养活",
+            "关系",
+            "先生",
+            "兄弟",
+            "便宜",
+            "使唤",
+            "佩服",
+            "作坊",
+            "体面",
+            "位置",
+            "似的",
+            "伙计",
+            "休息",
+            "什么",
+            "人家",
+            "亲戚",
+            "亲家",
+            "交情",
+            "云彩",
+            "事情",
+            "买卖",
+            "主意",
+            "丫头",
+            "丧气",
+            "两口",
+            "东西",
+            "东家",
+            "世故",
+            "不由",
+            "不在",
+            "下水",
+            "下巴",
+            "上头",
+            "上司",
+            "丈夫",
+            "丈人",
+            "一辈",
+            "那个",
+            "菩萨",
+            "父亲",
+            "母亲",
+            "咕噜",
+            "邋遢",
+            "费用",
+            "冤家",
+            "甜头",
+            "介绍",
+            "荒唐",
+            "大人",
+            "泥鳅",
+            "幸福",
+            "熟悉",
+            "计划",
+            "扑腾",
+            "蜡烛",
+            "姥爷",
+            "照顾",
+            "喉咙",
+            "吉他",
+            "弄堂",
+            "蚂蚱",
+            "凤凰",
+            "拖沓",
+            "寒碜",
+            "糟蹋",
+            "倒腾",
+            "报复",
+            "逻辑",
+            "盘缠",
+            "喽啰",
+            "牢骚",
+            "咖喱",
+            "扫把",
+            "惦记",
         }
         self.must_not_neural_tone_words = {
-            "男子", "女子", "分子", "原子", "量子", "莲子", "石子", "瓜子", "电子", "人人", "虎虎"
+            "男子",
+            "女子",
+            "分子",
+            "原子",
+            "量子",
+            "莲子",
+            "石子",
+            "瓜子",
+            "电子",
+            "人人",
+            "虎虎",
         }
         self.punc = "：，；。？！“”‘’':,;.?!"
 
@@ -72,14 +463,15 @@ class ToneSandhi():
     # word: "家里"
     # pos: "s"
     # finals: ['ia1', 'i3']
-    def _neural_sandhi(self, word: str, pos: str,
-                       finals: List[str]) -> List[str]:
-
+    def _neural_sandhi(self, word: str, pos: str, finals: List[str]) -> List[str]:
         # reduplication words for n. and v. e.g. 奶奶, 试试, 旺旺
         for j, item in enumerate(word):
-            if j - 1 >= 0 and item == word[j - 1] and pos[0] in {
-                "n", "v", "a"
-            } and word not in self.must_not_neural_tone_words:
+            if (
+                j - 1 >= 0
+                and item == word[j - 1]
+                and pos[0] in {"n", "v", "a"}
+                and word not in self.must_not_neural_tone_words
+            ):
                 finals[j] = finals[j][:-1] + "5"
         ge_idx = word.find("个")
         if len(word) >= 1 and word[-1] in "吧呢啊呐噻嘛吖嗨呐哦哒额滴哩哟喽啰耶喔诶":
@@ -89,9 +481,12 @@ class ToneSandhi():
         # e.g. 走了, 看着, 去过
         # elif len(word) == 1 and word in "了着过" and pos in {"ul", "uz", "ug"}:
         #     finals[-1] = finals[-1][:-1] + "5"
-        elif len(word) > 1 and word[-1] in "们子" and pos in {
-            "r", "n"
-        } and word not in self.must_not_neural_tone_words:
+        elif (
+            len(word) > 1
+            and word[-1] in "们子"
+            and pos in {"r", "n"}
+            and word not in self.must_not_neural_tone_words
+        ):
             finals[-1] = finals[-1][:-1] + "5"
         # e.g. 桌上, 地下, 家里
         elif len(word) > 1 and word[-1] in "上下里" and pos in {"s", "l", "f"}:
@@ -100,21 +495,26 @@ class ToneSandhi():
         elif len(word) > 1 and word[-1] in "来去" and word[-2] in "上下进出回过起开":
             finals[-1] = finals[-1][:-1] + "5"
         # 个做量词
-        elif (ge_idx >= 1 and
-              (word[ge_idx - 1].isnumeric() or
-               word[ge_idx - 1] in "几有两半多各整每做是")) or word == '个':
+        elif (
+            ge_idx >= 1
+            and (word[ge_idx - 1].isnumeric() or word[ge_idx - 1] in "几有两半多各整每做是")
+        ) or word == "个":
             finals[ge_idx] = finals[ge_idx][:-1] + "5"
         else:
-            if word in self.must_neural_tone_words or word[
-                                                      -2:] in self.must_neural_tone_words:
+            if (
+                word in self.must_neural_tone_words
+                or word[-2:] in self.must_neural_tone_words
+            ):
                 finals[-1] = finals[-1][:-1] + "5"
 
         word_list = self._split_word(word)
-        finals_list = [finals[:len(word_list[0])], finals[len(word_list[0]):]]
+        finals_list = [finals[: len(word_list[0])], finals[len(word_list[0]) :]]
         for i, word in enumerate(word_list):
             # conventional neural in Chinese
-            if word in self.must_neural_tone_words or word[
-                                                      -2:] in self.must_neural_tone_words:
+            if (
+                word in self.must_neural_tone_words
+                or word[-2:] in self.must_neural_tone_words
+            ):
                 finals_list[i][-1] = finals_list[i][-1][:-1] + "5"
         finals = sum(finals_list, [])
         return finals
@@ -126,17 +526,17 @@ class ToneSandhi():
         else:
             for i, char in enumerate(word):
                 # "不" before tone4 should be bu2, e.g. 不怕
-                if char == "不" and i + 1 < len(word) and finals[i +
-                                                                1][-1] == "4":
+                if char == "不" and i + 1 < len(word) and finals[i + 1][-1] == "4":
                     finals[i] = finals[i][:-1] + "2"
         return finals
 
     def _yi_sandhi(self, word: str, finals: List[str]) -> List[str]:
         # "一" in number sequences, e.g. 一零零, 二一零
         if word.find("一") != -1 and all(
-                [item.isnumeric() for item in word if item != "一"]):
+            [item.isnumeric() for item in word if item != "一"]
+        ):
             return finals
-        # "一" between reduplication words shold be yi5, e.g. 看一看
+        # "一" between reduplication words should be yi5, e.g. 看一看
         elif len(word) == 3 and word[1] == "一" and word[0] == word[-1]:
             finals[1] = finals[1][:-1] + "5"
         # when "一" is ordinal word, it should be yi1
@@ -161,10 +561,10 @@ class ToneSandhi():
         first_subword = word_list[0]
         first_begin_idx = word.find(first_subword)
         if first_begin_idx == 0:
-            second_subword = word[len(first_subword):]
+            second_subword = word[len(first_subword) :]
             new_word_list = [first_subword, second_subword]
         else:
-            second_subword = word[:-len(first_subword)]
+            second_subword = word[: -len(first_subword)]
             new_word_list = [second_subword, first_subword]
         return new_word_list
 
@@ -182,18 +582,19 @@ class ToneSandhi():
                 elif len(word_list[0]) == 1:
                     finals[1] = finals[1][:-1] + "2"
             else:
-                finals_list = [
-                    finals[:len(word_list[0])], finals[len(word_list[0]):]
-                ]
+                finals_list = [finals[: len(word_list[0])], finals[len(word_list[0]) :]]
                 if len(finals_list) == 2:
                     for i, sub in enumerate(finals_list):
                         # e.g. 所有/人
                         if self._all_tone_three(sub) and len(sub) == 2:
                             finals_list[i][0] = finals_list[i][0][:-1] + "2"
                         # e.g. 好/喜欢
-                        elif i == 1 and not self._all_tone_three(sub) and finals_list[i][0][-1] == "3" and \
-                                finals_list[0][-1][-1] == "3":
-
+                        elif (
+                            i == 1
+                            and not self._all_tone_three(sub)
+                            and finals_list[i][0][-1] == "3"
+                            and finals_list[0][-1][-1] == "3"
+                        ):
                             finals_list[0][-1] = finals_list[0][-1][:-1] + "2"
                         finals = sum(finals_list, [])
         # split idiom into two words who's length is 2
@@ -222,7 +623,7 @@ class ToneSandhi():
                 new_seg.append((word, pos))
             last_word = word[:]
         if last_word == "不":
-            new_seg.append((last_word, 'd'))
+            new_seg.append((last_word, "d"))
             last_word = ""
         return new_seg
 
@@ -236,12 +637,21 @@ class ToneSandhi():
         new_seg = []
         # function 1
         for i, (word, pos) in enumerate(seg):
-            if i - 1 >= 0 and word == "一" and i + 1 < len(seg) and seg[i - 1][
-                0] == seg[i + 1][0] and seg[i - 1][1] == "v":
+            if (
+                i - 1 >= 0
+                and word == "一"
+                and i + 1 < len(seg)
+                and seg[i - 1][0] == seg[i + 1][0]
+                and seg[i - 1][1] == "v"
+            ):
                 new_seg[i - 1][0] = new_seg[i - 1][0] + "一" + new_seg[i - 1][0]
             else:
-                if i - 2 >= 0 and seg[i - 1][0] == "一" and seg[i - 2][
-                    0] == word and pos == "v":
+                if (
+                    i - 2 >= 0
+                    and seg[i - 1][0] == "一"
+                    and seg[i - 2][0] == word
+                    and pos == "v"
+                ):
                     continue
                 else:
                     new_seg.append([word, pos])
@@ -257,22 +667,27 @@ class ToneSandhi():
 
     # the first and the second words are all_tone_three
     def _merge_continuous_three_tones(
-            self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
+        self, seg: List[Tuple[str, str]]
+    ) -> List[Tuple[str, str]]:
         new_seg = []
         sub_finals_list = [
-            lazy_pinyin(
-                word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
+            lazy_pinyin(word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
             for (word, pos) in seg
         ]
         assert len(sub_finals_list) == len(seg)
         merge_last = [False] * len(seg)
         for i, (word, pos) in enumerate(seg):
-            if i - 1 >= 0 and self._all_tone_three(
-                    sub_finals_list[i - 1]) and self._all_tone_three(
-                sub_finals_list[i]) and not merge_last[i - 1]:
+            if (
+                i - 1 >= 0
+                and self._all_tone_three(sub_finals_list[i - 1])
+                and self._all_tone_three(sub_finals_list[i])
+                and not merge_last[i - 1]
+            ):
                 # if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi
-                if not self._is_reduplication(seg[i - 1][0]) and len(
-                        seg[i - 1][0]) + len(seg[i][0]) <= 3:
+                if (
+                    not self._is_reduplication(seg[i - 1][0])
+                    and len(seg[i - 1][0]) + len(seg[i][0]) <= 3
+                ):
                     new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
                     merge_last[i] = True
                 else:
@@ -287,21 +702,27 @@ class ToneSandhi():
 
     # the last char of first word and the first char of second word is tone_three
     def _merge_continuous_three_tones_2(
-            self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
+        self, seg: List[Tuple[str, str]]
+    ) -> List[Tuple[str, str]]:
         new_seg = []
         sub_finals_list = [
-            lazy_pinyin(
-                word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
+            lazy_pinyin(word, neutral_tone_with_five=True, style=Style.FINALS_TONE3)
             for (word, pos) in seg
         ]
         assert len(sub_finals_list) == len(seg)
         merge_last = [False] * len(seg)
         for i, (word, pos) in enumerate(seg):
-            if i - 1 >= 0 and sub_finals_list[i - 1][-1][-1] == "3" and sub_finals_list[i][0][-1] == "3" and not \
-                    merge_last[i - 1]:
+            if (
+                i - 1 >= 0
+                and sub_finals_list[i - 1][-1][-1] == "3"
+                and sub_finals_list[i][0][-1] == "3"
+                and not merge_last[i - 1]
+            ):
                 # if the last word is reduplication, not merge, because reduplication need to be _neural_sandhi
-                if not self._is_reduplication(seg[i - 1][0]) and len(
-                        seg[i - 1][0]) + len(seg[i][0]) <= 3:
+                if (
+                    not self._is_reduplication(seg[i - 1][0])
+                    and len(seg[i - 1][0]) + len(seg[i][0]) <= 3
+                ):
                     new_seg[-1][0] = new_seg[-1][0] + seg[i][0]
                     merge_last[i] = True
                 else:
@@ -319,8 +740,7 @@ class ToneSandhi():
                 new_seg.append([word, pos])
         return new_seg
 
-    def _merge_reduplication(
-            self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
+    def _merge_reduplication(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
         new_seg = []
         for i, (word, pos) in enumerate(seg):
             if new_seg and word == new_seg[-1][0]:
@@ -329,8 +749,7 @@ class ToneSandhi():
                 new_seg.append([word, pos])
         return new_seg
 
-    def pre_merge_for_modify(
-            self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
+    def pre_merge_for_modify(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
         seg = self._merge_bu(seg)
         try:
             seg = self._merge_yi(seg)
@@ -342,8 +761,7 @@ class ToneSandhi():
         seg = self._merge_er(seg)
         return seg
 
-    def modified_tone(self, word: str, pos: str,
-                      finals: List[str]) -> List[str]:
+    def modified_tone(self, word: str, pos: str, finals: List[str]) -> List[str]:
         finals = self._bu_sandhi(word, finals)
         finals = self._yi_sandhi(word, finals)
         finals = self._neural_sandhi(word, pos, finals)

bert_vits2/utils.py

@@ -1,12 +1,6 @@
 import os
-import glob
 import sys
-import argparse
 import logging
-import json
-import subprocess
-import numpy as np
-from scipy.io.wavfile import read
 import torch
 
 MATPLOTLIB_FLAG = False
@@ -42,7 +36,13 @@ def load_checkpoint(checkpoint_path, model, optimizer=None, skip_optimizer=False
             new_state_dict[k] = saved_state_dict[k]
             assert saved_state_dict[k].shape == v.shape, (saved_state_dict[k].shape, v.shape)
         except:
-            print("error, %s is not in the checkpoint" % k)
+            # For upgrading from the old version
+            if "ja_bert_proj" in k:
+                v = torch.zeros_like(v)
+                logger.warning(
+                    f"If you are using an older version of the model, you should add the parameter \"legacy\":true to the data of the model's config.json")
+            logger.error(f"{k} is not in the checkpoint")
+
             new_state_dict[k] = v
     if hasattr(model, 'module'):
         model.module.load_state_dict(new_state_dict, strict=False)
@@ -52,241 +52,3 @@ def load_checkpoint(checkpoint_path, model, optimizer=None, skip_optimizer=False
     logger.info("Loaded checkpoint '{}' (iteration {})".format(
         checkpoint_path, iteration))
     return model, optimizer, learning_rate, iteration
-
-
-def save_checkpoint(model, optimizer, learning_rate, iteration, checkpoint_path):
-    logger.info("Saving model and optimizer state at iteration {} to {}".format(
-        iteration, checkpoint_path))
-    if hasattr(model, 'module'):
-        state_dict = model.module.state_dict()
-    else:
-        state_dict = model.state_dict()
-    torch.save({'model': state_dict,
-                'iteration': iteration,
-                'optimizer': optimizer.state_dict(),
-                'learning_rate': learning_rate}, checkpoint_path)
-
-
-def summarize(writer, global_step, scalars={}, histograms={}, images={}, audios={}, audio_sampling_rate=22050):
-    for k, v in scalars.items():
-        writer.add_scalar(k, v, global_step)
-    for k, v in histograms.items():
-        writer.add_histogram(k, v, global_step)
-    for k, v in images.items():
-        writer.add_image(k, v, global_step, dataformats='HWC')
-    for k, v in audios.items():
-        writer.add_audio(k, v, global_step, audio_sampling_rate)
-
-
-def latest_checkpoint_path(dir_path, regex="G_*.pth"):
-    f_list = glob.glob(os.path.join(dir_path, regex))
-    f_list.sort(key=lambda f: int("".join(filter(str.isdigit, f))))
-    x = f_list[-1]
-    print(x)
-    return x
-
-
-def plot_spectrogram_to_numpy(spectrogram):
-    global MATPLOTLIB_FLAG
-    if not MATPLOTLIB_FLAG:
-        import matplotlib
-        matplotlib.use("Agg")
-        MATPLOTLIB_FLAG = True
-        mpl_logger = logging.getLogger('matplotlib')
-        mpl_logger.setLevel(logging.WARNING)
-    import matplotlib.pylab as plt
-    import numpy as np
-
-    fig, ax = plt.subplots(figsize=(10, 2))
-    im = ax.imshow(spectrogram, aspect="auto", origin="lower",
-                   interpolation='none')
-    plt.colorbar(im, ax=ax)
-    plt.xlabel("Frames")
-    plt.ylabel("Channels")
-    plt.tight_layout()
-
-    fig.canvas.draw()
-    data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
-    data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
-    plt.close()
-    return data
-
-
-def plot_alignment_to_numpy(alignment, info=None):
-    global MATPLOTLIB_FLAG
-    if not MATPLOTLIB_FLAG:
-        import matplotlib
-        matplotlib.use("Agg")
-        MATPLOTLIB_FLAG = True
-        mpl_logger = logging.getLogger('matplotlib')
-        mpl_logger.setLevel(logging.WARNING)
-    import matplotlib.pylab as plt
-    import numpy as np
-
-    fig, ax = plt.subplots(figsize=(6, 4))
-    im = ax.imshow(alignment.transpose(), aspect='auto', origin='lower',
-                   interpolation='none')
-    fig.colorbar(im, ax=ax)
-    xlabel = 'Decoder timestep'
-    if info is not None:
-        xlabel += '\n\n' + info
-    plt.xlabel(xlabel)
-    plt.ylabel('Encoder timestep')
-    plt.tight_layout()
-
-    fig.canvas.draw()
-    data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
-    data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
-    plt.close()
-    return data
-
-
-def load_wav_to_torch(full_path):
-    sampling_rate, data = read(full_path)
-    return torch.FloatTensor(data.astype(np.float32)), sampling_rate
-
-
-def load_filepaths_and_text(filename, split="|"):
-    with open(filename, encoding='utf-8') as f:
-        filepaths_and_text = [line.strip().split(split) for line in f]
-    return filepaths_and_text
-
-
-def get_hparams(init=True):
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-c', '--config', type=str, default="./configs/base.json",
-                        help='JSON file for configuration')
-    parser.add_argument('-m', '--model', type=str, required=True,
-                        help='Model name')
-
-    args = parser.parse_args()
-    model_dir = os.path.join("./logs", args.model)
-
-    if not os.path.exists(model_dir):
-        os.makedirs(model_dir)
-
-    config_path = args.config
-    config_save_path = os.path.join(model_dir, "config.json")
-    if init:
-        with open(config_path, "r") as f:
-            data = f.read()
-        with open(config_save_path, "w") as f:
-            f.write(data)
-    else:
-        with open(config_save_path, "r") as f:
-            data = f.read()
-    config = json.loads(data)
-
-    hparams = HParams(**config)
-    hparams.model_dir = model_dir
-    return hparams
-
-
-def clean_checkpoints(path_to_models='logs/44k/', n_ckpts_to_keep=2, sort_by_time=True):
-    """Freeing up space by deleting saved ckpts
-
-  Arguments:
-  path_to_models    --  Path to the model directory
-  n_ckpts_to_keep   --  Number of ckpts to keep, excluding G_0.pth and D_0.pth
-  sort_by_time      --  True -> chronologically delete ckpts
-                        False -> lexicographically delete ckpts
-  """
-    import re
-    ckpts_files = [f for f in os.listdir(path_to_models) if os.path.isfile(os.path.join(path_to_models, f))]
-    name_key = (lambda _f: int(re.compile('._(\d+)\.pth').match(_f).group(1)))
-    time_key = (lambda _f: os.path.getmtime(os.path.join(path_to_models, _f)))
-    sort_key = time_key if sort_by_time else name_key
-    x_sorted = lambda _x: sorted([f for f in ckpts_files if f.startswith(_x) and not f.endswith('_0.pth')],
-                                 key=sort_key)
-    to_del = [os.path.join(path_to_models, fn) for fn in
-              (x_sorted('G')[:-n_ckpts_to_keep] + x_sorted('D')[:-n_ckpts_to_keep])]
-    del_info = lambda fn: logger.info(f".. Free up space by deleting ckpt {fn}")
-    del_routine = lambda x: [os.remove(x), del_info(x)]
-    rs = [del_routine(fn) for fn in to_del]
-
-
-def get_hparams_from_dir(model_dir):
-    config_save_path = os.path.join(model_dir, "config.json")
-    with open(config_save_path, "r", encoding='utf-8') as f:
-        data = f.read()
-    config = json.loads(data)
-
-    hparams = HParams(**config)
-    hparams.model_dir = model_dir
-    return hparams
-
-
-def get_hparams_from_file(config_path):
-    with open(config_path, "r", encoding='utf-8') as f:
-        data = f.read()
-    config = json.loads(data)
-
-    hparams = HParams(**config)
-    return hparams
-
-
-def check_git_hash(model_dir):
-    source_dir = os.path.dirname(os.path.realpath(__file__))
-    if not os.path.exists(os.path.join(source_dir, ".git")):
-        logger.warn("{} is not a git repository, therefore hash value comparison will be ignored.".format(
-            source_dir
-        ))
-        return
-
-    cur_hash = subprocess.getoutput("git rev-parse HEAD")
-
-    path = os.path.join(model_dir, "githash")
-    if os.path.exists(path):
-        saved_hash = open(path).read()
-        if saved_hash != cur_hash:
-            logger.warn("git hash values are different. {}(saved) != {}(current)".format(
-                saved_hash[:8], cur_hash[:8]))
-    else:
-        open(path, "w").write(cur_hash)
-
-
-def get_logger(model_dir, filename="train.log"):
-    global logger
-    logger = logging.getLogger(os.path.basename(model_dir))
-    logger.setLevel(logging.DEBUG)
-
-    formatter = logging.Formatter("%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s")
-    if not os.path.exists(model_dir):
-        os.makedirs(model_dir)
-    h = logging.FileHandler(os.path.join(model_dir, filename))
-    h.setLevel(logging.DEBUG)
-    h.setFormatter(formatter)
-    logger.addHandler(h)
-    return logger
-
-
-class HParams():
-    def __init__(self, **kwargs):
-        for k, v in kwargs.items():
-            if type(v) == dict:
-                v = HParams(**v)
-            self[k] = v
-
-    def keys(self):
-        return self.__dict__.keys()
-
-    def items(self):
-        return self.__dict__.items()
-
-    def values(self):
-        return self.__dict__.values()
-
-    def __len__(self):
-        return len(self.__dict__)
-
-    def __getitem__(self, key):
-        return getattr(self, key)
-
-    def __setitem__(self, key, value):
-        return setattr(self, key, value)
-
-    def __contains__(self, key):
-        return key in self.__dict__
-
-    def __repr__(self):
-        return self.__dict__.__repr__()

docker-compose-gpu.yaml

@@ -8,7 +8,7 @@ services:
       - 23456:23456
     environment:
       LANG: 'C.UTF-8'
-      TZ: Asia/Shanghai #timezone
+      #TZ: Asia/Shanghai #timezone
     volumes:
       - ./Model:/app/Model # 挂载模型文件夹
       - ./config.py:/app/config.py # 挂载配置文件
@@ -17,5 +17,10 @@ services:
       - ./vits/bert:/app/vits/bert # vits_chinese
       - ./bert_vits2/bert/chinese-roberta-wwm-ext-large:/app/bert_vits2/bert/chinese-roberta-wwm-ext-large # Bert-vits2
       - ./pyopenjtalk/open_jtalk_dic_utf_8-1.11:/usr/local/lib/python3.10/site-packages/pyopenjtalk/open_jtalk_dic_utf_8-1.11 #pyopentjalk
-    devices:
-      - "/dev/nvidia0:/dev/nvidia0"
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              #device_ids: ['0', '3']
+              capabilities: [gpu]

requirements.txt

@@ -1,16 +1,19 @@
-numba
-librosa
+# Core scientific and computation libraries
 numpy==1.23.3
 scipy
-MarkupSafe==2.1.2
-torch
+
+# Audio processing
+librosa
+soundfile==0.12.1
+numba
+graiax-silkcoder[libsndfile]
+
+# Natural Language Processing and Text Conversion
 unidecode
-six==1.16.0
-openjtalk==0.3.0.dev2
+pyopenjtalk==0.3.2
 jamo
 pypinyin
 jieba
-protobuf
 cn2an
 inflect
 eng_to_ipa
@@ -18,14 +21,30 @@ ko_pron
 indic_transliteration
 num_thai
 opencc
-audonnx
-flask==2.2.3
-soundfile==0.12.1
-graiax-silkcoder[libsndfile]
-flask_apscheduler
 fasttext
 fastlid
 langid
 phonemizer==3.2.1
 transformers
-pydantic==2.3.0
+num2words
+mecab-python3
+unidic-lite
+
+# Machine Learning and Deep Learning
+torch
+audonnx
+
+# Web and API services
+flask==2.2.3
+flask_apscheduler
+pydantic==2.3.0
+werkzeug==2.3.6
+
+# Compression and Decompression
+py7zr
+
+# Other utilities and dependencies
+MarkupSafe==2.1.2
+six==1.16.0
+protobuf
+tqdm

templates/index.html

@@ -532,4 +532,4 @@
     });
 </script>
 </body>
-</html>
+</html>

utils/__init__.py

@@ -0,0 +1,3 @@
+from utils.classify_language import classify_language
+from utils.data_utils import get_hparams_from_file, load_checkpoint, load_audio_to_torch
+from utils.lang_dict import lang_dict

utils/classify_language.py

@@ -0,0 +1,49 @@
+from config import LANGUAGE_IDENTIFICATION_LIBRARY
+
+module = LANGUAGE_IDENTIFICATION_LIBRARY.lower()
+
+
+def classify_language(text: str, target_languages: list = None) -> str:
+    if module == "fastlid" or module == "fasttext":
+        from fastlid import fastlid
+        classifier = fastlid
+        if target_languages != None: fastlid.set_languages = target_languages
+    elif module == "langid":
+        import langid
+        classifier = langid.classify
+        if target_languages != None: langid.set_languages(target_languages)
+    else:
+        raise ValueError(f"Wrong LANGUAGE_IDENTIFICATION_LIBRARY in config.py")
+
+    lang = classifier(text)[0]
+
+    return lang
+
+
+def classify_zh_ja(text: str) -> str:
+    for idx, char in enumerate(text):
+        unicode_val = ord(char)
+
+        # 检测日语字符
+        if 0x3040 <= unicode_val <= 0x309F or 0x30A0 <= unicode_val <= 0x30FF:
+            return "ja"
+
+        # 检测汉字字符
+        if 0x4E00 <= unicode_val <= 0x9FFF:
+            # 检查周围的字符
+            next_char = text[idx + 1] if idx + 1 < len(text) else None
+
+            if next_char and (0x3040 <= ord(next_char) <= 0x309F or 0x30A0 <= ord(next_char) <= 0x30FF):
+                return "ja"
+
+    return "zh"
+
+
+if __name__ == "__main__":
+    text = "这是一个测试文本"
+    print(classify_language(text))
+    print(classify_zh_ja(text))  # "zh"
+
+    text = "これはテストテキストです"
+    print(classify_language(text))
+    print(classify_zh_ja(text))  # "ja"

utils/data_utils.py

@@ -0,0 +1,97 @@
+import os
+import logging
+
+
+class HParams():
+    def __init__(self, **kwargs):
+        for k, v in kwargs.items():
+            if type(v) == dict:
+                v = HParams(**v)
+            self[k] = v
+
+    def keys(self):
+        return self.__dict__.keys()
+
+    def items(self):
+        return self.__dict__.items()
+
+    def values(self):
+        return self.__dict__.values()
+
+    def __len__(self):
+        return len(self.__dict__)
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        return setattr(self, key, value)
+
+    def __contains__(self, key):
+        return key in self.__dict__
+
+    def __repr__(self):
+        return self.__dict__.__repr__()
+
+
+def load_checkpoint(checkpoint_path, model):
+    from torch import load
+    checkpoint_dict = load(checkpoint_path, map_location='cpu')
+    iteration = checkpoint_dict.get('iteration', None)
+    saved_state_dict = checkpoint_dict['model']
+    if hasattr(model, 'module'):
+        state_dict = model.module.state_dict()
+    else:
+        state_dict = model.state_dict()
+    new_state_dict = {}
+    for k, v in state_dict.items():
+        try:
+            new_state_dict[k] = saved_state_dict[k]
+        except:
+            logging.info(f"{k} is not in the checkpoint")
+            new_state_dict[k] = v
+    if hasattr(model, 'module'):
+        model.module.load_state_dict(new_state_dict)
+    else:
+        model.load_state_dict(new_state_dict)
+    if iteration:
+        logging.info(f"Loaded checkpoint '{checkpoint_path}' (iteration {iteration})")
+    else:
+        logging.info(f"Loaded checkpoint '{checkpoint_path}'")
+    return
+
+
+def get_hparams_from_file(config_path):
+    from json import loads
+    with open(config_path, 'r', encoding='utf-8') as f:
+        data = f.read()
+    config = loads(data)
+
+    hparams = HParams(**config)
+    return hparams
+
+
+def load_audio_to_torch(full_path, target_sampling_rate):
+    import librosa
+    from torch import FloatTensor
+    from numpy import float32
+    audio, sampling_rate = librosa.load(full_path, sr=target_sampling_rate, mono=True)
+    return FloatTensor(audio.astype(float32))
+
+
+def clean_folder(folder_path):
+    for filename in os.listdir(folder_path):
+        file_path = os.path.join(folder_path, filename)
+        # 如果是文件，则删除文件。如果是文件夹则跳过。
+        if os.path.isfile(file_path):
+            os.remove(file_path)
+
+
+def check_is_none(item) -> bool:
+    # none -> True, not none -> False
+    return item is None or (isinstance(item, str) and str(item).isspace()) or str(item) == ""
+
+
+def save_audio(audio, path):
+    with open(path, "wb") as f:
+        f.write(audio)

utils/download.py

@@ -0,0 +1,92 @@
+import os
+import hashlib
+import tarfile
+import urllib.request
+import zipfile
+
+from tqdm import tqdm
+from pathlib import Path
+from logger import logger
+from py7zr import SevenZipFile
+
+
+class TqdmUpTo(tqdm):
+    def update_to(self, b=1, bsize=1, tsize=None):
+        if tsize is not None:
+            self.total = tsize
+        self.update(b * bsize - self.n)
+
+
+def download_file(url, dest_path):
+    with TqdmUpTo(unit="B", unit_scale=True, unit_divisor=1024, miniters=1, desc=url.split('/')[-1]) as t:
+        urllib.request.urlretrieve(url, dest_path, reporthook=t.update_to)
+
+
+def verify_md5(file_path, expected_md5):
+    md5 = hashlib.md5(file_path.read_bytes()).hexdigest()
+    if md5 != expected_md5:
+        return False, f"MD5 mismatch: {md5} != {expected_md5}"
+    return True, ""
+
+
+def extract_file(file_path, destination=None):
+    """
+    Extract a compressed file based on its extension.
+    If destination is not specified, it will be extracted to its parent directory.
+    """
+    if destination is None:
+        destination = Path(file_path).parent
+
+    if file_path.endswith('.zip'):
+        with zipfile.ZipFile(file_path, 'r') as zip_ref:
+            zip_ref.extractall(destination)
+    elif file_path.endswith('.tar.gz'):
+        with tarfile.open(file_path, 'r:gz') as tar_ref:
+            tar_ref.extractall(destination)
+    elif file_path.endswith('.tar.bz2'):
+        with tarfile.open(file_path, 'r:bz2') as tar_ref:
+            tar_ref.extractall(destination)
+    elif file_path.endswith('.7z'):
+        with SevenZipFile(file_path, mode='r') as z:
+            z.extractall(destination)
+    else:
+        print(f"Unsupported compression format for file {file_path}")
+
+
+def download_and_verify(urls, target_path, expected_md5=None, extract_destination=None):
+    for url in urls:
+        try:
+            download_file(url, target_path)
+            break
+        except Exception as error:
+            logger.error(f"downloading from URL {url}: {error}")
+
+    else:  # This else is tied to the for loop, and executes if no download is successful
+        return False, "Error downloading from all provided URLs."
+
+    if expected_md5 is not None:
+        success, message = verify_md5(Path(target_path), expected_md5)
+        if not success:
+            os.remove(target_path)
+            return False, message
+
+    # If it's a compressed file, extract it
+    if target_path.endswith(('.zip', '.tar.gz', '.tar.bz2', '.7z')):
+        extract_file(target_path, extract_destination)
+        os.remove(target_path)
+
+    return True, "File downloaded, verified, and extracted successfully!"
+
+
+if __name__ == "__main__":
+    URLS = [
+        "YOUR_PRIMARY_URL_HERE",
+        "YOUR_FIRST_BACKUP_URL_HERE",
+        # ... you can add more backup URLs as needed
+    ]
+    TARGET_PATH = ""
+    EXPECTED_MD5 = ""
+    EXTRACT_DESTINATION = ""
+
+    success, message = download_and_verify(URLS, TARGET_PATH, EXPECTED_MD5, EXTRACT_DESTINATION)
+    print(message)

utils/lang_dict.py

@@ -0,0 +1,22 @@
+lang_dict = {
+    "english_cleaners": ["en"],
+    "english_cleaners2": ["en"],
+    "japanese_cleaners": ["ja"],
+    "japanese_cleaners2": ["ja"],
+    "korean_cleaners": ["ko"],
+    "chinese_cleaners": ["zh"],
+    "zh_ja_mixture_cleaners": ["zh", "ja"],
+    "sanskrit_cleaners": ["sa"],
+    "cjks_cleaners": ["zh", "ja", "ko", "sa"],
+    "cjke_cleaners": ["zh", "ja", "ko", "en"],
+    "cjke_cleaners2": ["zh", "ja", "ko", "en"],
+    "cje_cleaners": ["zh", "ja", "en"],
+    "cje_cleaners2": ["zh", "ja", "en"],
+    "thai_cleaners": ["th"],
+    "shanghainese_cleaners": ["sh"],
+    "chinese_dialect_cleaners": ["zh", "ja", "sh", "gd", "en", "SZ", "WX", "CZ", "HZ", "SX", "NB", "JJ", "YX", "JD",
+                                 "ZR", "PH", "TX", "JS", "HN", "LP", "XS", "FY", "RA", "CX", "SM", "TT", "WZ", "SC",
+                                 "YB"],
+    "bert_chinese_cleaners": ["zh"],
+    "bert_vits2": ["zh", "ja"]
+}

utils/load_model.py

@@ -0,0 +1,172 @@
+import os
+import json
+import logging
+import config
+import numpy as np
+from utils.data_utils import check_is_none
+from vits import VITS
+from voice import TTS
+from config import DEVICE as device
+from utils.lang_dict import lang_dict
+
+
+
+def recognition_model_type(model_config_json: json) -> str:
+    model_config = json.load(model_config_json)
+    symbols = model_config.get("symbols", None)
+    emotion_embedding = model_config.get("data").get("emotion_embedding", False)
+
+    if "use_spk_conditioned_encoder" in model_config.get("model"):
+        model_type = "bert_vits2"
+        return model_type
+
+    if symbols != None:
+        if not emotion_embedding:
+            mode_type = "vits"
+        else:
+            mode_type = "w2v2"
+    else:
+        mode_type = "hubert"
+
+    return mode_type
+
+
+def load_npy(emotion_reference_npy):
+    if isinstance(emotion_reference_npy, list):
+        # check if emotion_reference_npy is endwith .npy
+        for i in emotion_reference_npy:
+            model_extention = os.path.splitext(i)[1]
+            if model_extention != ".npy":
+                raise ValueError(f"Unsupported model type: {model_extention}")
+
+        # merge npy files
+        emotion_reference = np.empty((0, 1024))
+        for i in emotion_reference_npy:
+            tmp = np.load(i).reshape(-1, 1024)
+            emotion_reference = np.append(emotion_reference, tmp, axis=0)
+
+    elif os.path.isdir(emotion_reference_npy):
+        emotion_reference = np.empty((0, 1024))
+        for root, dirs, files in os.walk(emotion_reference_npy):
+            for file_name in files:
+                # check if emotion_reference_npy is endwith .npy
+                model_extention = os.path.splitext(file_name)[1]
+                if model_extention != ".npy":
+                    continue
+                file_path = os.path.join(root, file_name)
+
+                # merge npy files
+                tmp = np.load(file_path).reshape(-1, 1024)
+                emotion_reference = np.append(emotion_reference, tmp, axis=0)
+
+    elif os.path.isfile(emotion_reference_npy):
+        # check if emotion_reference_npy is endwith .npy
+        model_extention = os.path.splitext(emotion_reference_npy)[1]
+        if model_extention != ".npy":
+            raise ValueError(f"Unsupported model type: {model_extention}")
+
+        emotion_reference = np.load(emotion_reference_npy)
+    logging.info(f"Loaded emotional dimention npy range:{len(emotion_reference)}")
+    return emotion_reference
+
+
+def parse_models(model_list):
+    categorized_models = {
+        "vits": [],
+        "hubert": [],
+        "w2v2": [],
+        "bert_vits2": []
+    }
+
+    for model_info in model_list:
+        config_path = model_info[1]
+        with open(config_path, 'r', encoding='utf-8') as model_config:
+            model_type = recognition_model_type(model_config)
+        if model_type in categorized_models:
+            categorized_models[model_type].append(model_info)
+
+    return categorized_models
+
+
+def merge_models(model_list, model_class, model_type, additional_arg=None):
+    id_mapping_objs = []
+    speakers = []
+    new_id = 0
+
+    for obj_id, (model_path, config_path) in enumerate(model_list):
+        obj_args = {
+            "model": model_path,
+            "config": config_path,
+            "model_type": model_type,
+            "device": device
+        }
+        if additional_arg:
+            obj_args.update(additional_arg)
+
+        obj = model_class(**obj_args)
+
+        if model_type == "bert_vits2":
+            key = model_type
+        else:
+            key = obj.get_cleaner()
+
+        lang = lang_dict.get(key, ["unknown"])
+
+        for real_id, name in enumerate(obj.get_speakers()):
+            id_mapping_objs.append([real_id, obj, obj_id])
+            speakers.append({"id": new_id, "name": name, "lang": lang})
+            new_id += 1
+
+    return id_mapping_objs, speakers
+
+
+def load_model(model_list) -> TTS:
+    categorized_models = parse_models(model_list)
+
+    # Handle VITS
+    vits_objs, vits_speakers = merge_models(categorized_models["vits"], VITS, "vits")
+
+    # Handle HUBERT-VITS
+    hubert_vits_objs, hubert_vits_speakers = [], []
+    if len(categorized_models["hubert"]) != 0:
+        if getattr(config, "HUBERT_SOFT_MODEL", None) is None or check_is_none(config.HUBERT_SOFT_MODEL):
+            raise ValueError(f"Please configure HUBERT_SOFT_MODEL path in config.py")
+        try:
+            from vits.hubert_model import hubert_soft
+            hubert = hubert_soft(config.HUBERT_SOFT_MODEL)
+        except Exception as e:
+            raise ValueError(f"Load HUBERT_SOFT_MODEL failed {e}")
+
+        hubert_vits_objs, hubert_vits_speakers = merge_models(categorized_models["hubert"], VITS, "hubert",
+                                                              additional_arg={"additional_model": hubert})
+
+    # Handle W2V2-VITS
+    w2v2_vits_objs, w2v2_vits_speakers = [], []
+    w2v2_emotion_count = 0
+    if len(categorized_models["w2v2"]) != 0:
+        if getattr(config, "DIMENSIONAL_EMOTION_NPY", None) is None or check_is_none(
+                config.DIMENSIONAL_EMOTION_NPY):
+            raise ValueError(f"Please configure DIMENSIONAL_EMOTION_NPY path in config.py")
+        try:
+            emotion_reference = load_npy(config.DIMENSIONAL_EMOTION_NPY)
+        except Exception as e:
+            emotion_reference = None
+            raise ValueError(f"Load DIMENSIONAL_EMOTION_NPY failed {e}")
+
+        w2v2_vits_objs, w2v2_vits_speakers = merge_models(categorized_models["w2v2"], VITS, "w2v2",
+                                                          additional_arg={"additional_model": emotion_reference})
+        w2v2_emotion_count = len(emotion_reference) if emotion_reference is not None else 0
+
+    # Handle BERT-VITS2
+    bert_vits2_objs, bert_vits2_speakers = [], []
+    if len(categorized_models["bert_vits2"]) != 0:
+        from bert_vits2 import Bert_VITS2
+        bert_vits2_objs, bert_vits2_speakers = merge_models(categorized_models["bert_vits2"], Bert_VITS2, "bert_vits2")
+
+    voice_obj = {"VITS": vits_objs, "HUBERT-VITS": hubert_vits_objs, "W2V2-VITS": w2v2_vits_objs,
+                 "BERT-VITS2": bert_vits2_objs}
+    voice_speakers = {"VITS": vits_speakers, "HUBERT-VITS": hubert_vits_speakers, "W2V2-VITS": w2v2_vits_speakers,
+                      "BERT-VITS2": bert_vits2_speakers}
+
+    tts = TTS(voice_obj, voice_speakers, device=device, w2v2_emotion_count=w2v2_emotion_count)
+    return tts

utils/sentence.py

@@ -0,0 +1,91 @@
+import regex as re
+
+from logger import logger
+from utils.data_utils import check_is_none
+from utils.classify_language import classify_language
+
+
+def markup_language_type(text: str, target_languages: list = None) -> str:
+    pattern = r'[\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\>\=\?\@\[\]\{\}\\\\\^\_\`' \
+              r'\！？｡＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」' \
+              r'『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘\'\‛\“\”\„\‟…‧﹏.]+'
+    sentences = re.split(pattern, text)
+
+    pre_lang = ""
+    p = 0
+
+    for sentence in sentences:
+
+        if check_is_none(sentence): continue
+
+        lang = classify_language(sentence, target_languages)
+
+        if pre_lang == "":
+            text = text[:p] + text[p:].replace(sentence, f"[{lang.upper()}]{sentence}", 1)
+            p += len(f"[{lang.upper()}]")
+        elif pre_lang != lang:
+            text = text[:p] + text[p:].replace(sentence, f"[{pre_lang.upper()}][{lang.upper()}]{sentence}", 1)
+            p += len(f"[{pre_lang.upper()}][{lang.upper()}]")
+        pre_lang = lang
+        p += text[p:].index(sentence) + len(sentence)
+    text += f"[{pre_lang.upper()}]"
+
+    return text
+
+
+def cut(text: str, max: int) -> list:
+    pattern = r'[!(),—+\-.:;?？。，、；：]+'
+    sentences = re.split(pattern, text)
+    discarded_chars = re.findall(pattern, text)
+
+    sentence_list, count, p = [], 0, 0
+
+    # 按被分割的符号遍历
+    for i, discarded_chars in enumerate(discarded_chars):
+        count += len(sentences[i]) + len(discarded_chars)
+        if count >= max:
+            sentence_list.append(text[p:p + count].strip())
+            p += count
+            count = 0
+
+    # 加入最后剩余的文本
+    if p < len(text):
+        sentence_list.append(text[p:])
+
+    return sentence_list
+
+
+def sentence_split_and_markup(text, max=50, lang="auto", speaker_lang=None):
+    # 如果该speaker只支持一种语言
+    if speaker_lang is not None and len(speaker_lang) == 1:
+        if lang.upper() not in ["AUTO", "MIX"] and lang.lower() != speaker_lang[0]:
+            logger.debug(
+                f"lang \"{lang}\" is not in speaker_lang {speaker_lang},automatically set lang={speaker_lang[0]}")
+        lang = speaker_lang[0]
+
+    sentence_list = []
+    if lang.upper() != "MIX":
+        if max <= 0:
+            sentence_list.append(
+                markup_language_type(text,
+                                     speaker_lang) if lang.upper() == "AUTO" else f"[{lang.upper()}]{text}[{lang.upper()}]")
+        else:
+            for i in cut(text, max):
+                if check_is_none(i): continue
+                sentence_list.append(
+                    markup_language_type(i,
+                                         speaker_lang) if lang.upper() == "AUTO" else f"[{lang.upper()}]{i}[{lang.upper()}]")
+    else:
+        sentence_list.append(text)
+
+    for i in sentence_list:
+        logger.debug(i)
+
+    return sentence_list
+
+
+if __name__ == '__main__':
+    text = "这几天心里颇不宁静。今晚在院子里坐着乘凉，忽然想起日日走过的荷塘，在这满月的光里，总该另有一番样子吧。月亮渐渐地升高了，墙外马路上孩子们的欢笑，已经听不见了；妻在屋里拍着闰儿，迷迷糊糊地哼着眠歌。我悄悄地披了大衫，带上门出去。"
+    print(markup_language_type(text, languages=None))
+    print(cut(text, max=50))
+    print(sentence_split_and_markup(text, max=50, lang="auto", speaker_lang=None))

vits-simple-api-installer-latest.sh

@@ -228,19 +228,6 @@ read -p "${MESSAGES["ENTER_CHOICE"]}" choice_download_bert_vits2
 if [ "$choice_download_bert_vits2" -eq 1 ]; then
   mkdir -p bert_vits2/bert/chinese-roberta-wwm-ext-large
 
-  EXPECTED_MD5="78ef42421495cb23372bef9d069a75f3"
-  FILE_PATH="bert_vits2/bert/chinese-roberta-wwm-ext-large/flax_model.msgpack"
-  echo -e "${MESSAGES["VERIFYING"]}$FILE_PATH"
-  ACTUAL_MD5=$(md5sum $FILE_PATH | awk '{print $1}')
-
-  if [ "$EXPECTED_MD5" == "$ACTUAL_MD5" ]; then
-    echo "${MESSAGES["DOWNLOADED"]}"
-  else
-    echo "${MESSAGES["CORRUPTED"]}"
-    download_with_fallback bert_vits2/bert/chinese-roberta-wwm-ext-large/flax_model.msgpack \
-      "https://huggingface.co/hfl/chinese-roberta-wwm-ext-large/resolve/main/flax_model.msgpack"
-  fi
-
   EXPECTED_MD5="15d7435868fef1bd4222ff7820149a2a"
   FILE_PATH="bert_vits2/bert/chinese-roberta-wwm-ext-large/pytorch_model.bin"
   echo -e "${MESSAGES["VERIFYING"]}$FILE_PATH"
@@ -254,19 +241,6 @@ if [ "$choice_download_bert_vits2" -eq 1 ]; then
       "https://huggingface.co/hfl/chinese-roberta-wwm-ext-large/resolve/main/pytorch_model.bin"
   fi
 
-  EXPECTED_MD5="d15991416bd4a86fa127c70d3c0f4779"
-  FILE_PATH="bert_vits2/bert/chinese-roberta-wwm-ext-large/tf_model.h5"
-  echo -e "${MESSAGES["VERIFYING"]}$FILE_PATH"
-  ACTUAL_MD5=$(md5sum $FILE_PATH | awk '{print $1}')
-
-  if [ "$EXPECTED_MD5" == "$ACTUAL_MD5" ]; then
-    echo "${MESSAGES["DOWNLOADED"]}"
-  else
-    echo "${MESSAGES["CORRUPTED"]}"
-    download_with_fallback bert_vits2/bert/chinese-roberta-wwm-ext-large/tf_model.h5 \
-      "https://huggingface.co/hfl/chinese-roberta-wwm-ext-large/resolve/main/tf_model.h5"
-  fi
-
 fi
 
 if [ "$choice_gpu" -eq 2 ]; then

vits/bert/__init__.py

@@ -1,2 +1,15 @@
 """ from https://github.com/PlayVoice/vits_chinese """
-from .ProsodyModel import TTSProsody
+import os
+
+import config
+from utils.download import download_and_verify
+from .ProsodyModel import TTSProsody
+
+URLS = [
+    "https://huggingface.co/spaces/maxmax20160403/vits_chinese/resolve/main/bert/prosody_model.pt",
+]
+TARGET_PATH = os.path.join(config.ABS_PATH, "vits/bert/prosody_model.pt")
+EXPECTED_MD5 = None
+
+if not os.path.exists(TARGET_PATH):
+    success, message = download_and_verify(URLS, TARGET_PATH, EXPECTED_MD5)

vits/text/cantonese.py

@@ -1,9 +1,23 @@
+import os.path
 import re
 import cn2an
 import opencc
 import config
+from utils.download import download_and_verify
 
-converter = opencc.OpenCC(config.ABS_PATH + '/vits/text/chinese_dialect_lexicons/jyutjyu_2')
+URLS = [
+    "https://github.com/CjangCjengh/chinese-dialect-lexicons/releases/download/v1.0.3/chinese_dialects.7z",
+    "https://ghproxy.com/https://github.com/CjangCjengh/chinese-dialect-lexicons/releases/download/v1.0.3/chinese_dialects.7z",
+]
+TARGET_PATH = os.path.join(config.ABS_PATH, "vits/text/chinese_dialects.7z")
+EXTRACT_DESTINATION = os.path.join(config.ABS_PATH, "vits/text/chinese_dialect_lexicons/")
+EXPECTED_MD5 = None
+OPENCC_FILE_PATH = os.path.join(config.ABS_PATH, "vits/text/chinese_dialect_lexicons/jyutjyu.json")
+
+if not os.path.exists(OPENCC_FILE_PATH):
+    success, message = download_and_verify(URLS, TARGET_PATH, EXPECTED_MD5, EXTRACT_DESTINATION)
+    
+converter = opencc.OpenCC(OPENCC_FILE_PATH)
 
 # List of (Latin alphabet, ipa) pairs:
 _latin_to_ipa = [(re.compile('%s' % x[0]), x[1]) for x in [

vits/text/shanghainese.py

@@ -1,9 +1,23 @@
+import os
 import re
 import cn2an
 import opencc
 import config
+from utils.download import download_and_verify
 
-converter = opencc.OpenCC(config.ABS_PATH + '/vits/text/chinese_dialect_lexicons/zaonhe')
+URLS = [
+    "https://github.com/CjangCjengh/chinese-dialect-lexicons/releases/download/v1.0.3/chinese_dialects.7z",
+    "https://ghproxy.com/https://github.com/CjangCjengh/chinese-dialect-lexicons/releases/download/v1.0.3/chinese_dialects.7z",
+]
+TARGET_PATH = os.path.join(config.ABS_PATH, "vits/text/chinese_dialects.7z")
+EXTRACT_DESTINATION = os.path.join(config.ABS_PATH, "vits/text/chinese_dialect_lexicons/")
+EXPECTED_MD5 = None
+OPENCC_FILE_PATH = os.path.join(config.ABS_PATH, "vits/text/chinese_dialect_lexicons/zaonhe.json")
+
+if not os.path.exists(OPENCC_FILE_PATH):
+    success, message = download_and_verify(URLS, TARGET_PATH, EXPECTED_MD5, EXTRACT_DESTINATION)
+
+converter = opencc.OpenCC(OPENCC_FILE_PATH)
 
 # List of (Latin alphabet, ipa) pairs:
 _latin_to_ipa = [(re.compile('%s' % x[0]), x[1]) for x in [

vits/text/vits_pinyin.py

@@ -1,3 +1,4 @@
+""" from https://github.com/PlayVoice/vits_chinese """
 import pypinyin
 from pypinyin.contrib.neutral_tone import NeutralToneWith5Mixin
 from pypinyin.converter import DefaultConverter
@@ -41,12 +42,12 @@ class VITS_PinYin:
     def chinese_to_phonemes(self, text):
         # 考虑使用g2pw的chinese bert替换原始的pypinyin,目前测试下来运行速度太慢。
         # 将标准中文文本符号替换成 bert 符号库中的单符号,以保证bert的效果.
-        text = text.replace("——", "...")\
-            .replace("—", "...")\
-            .replace("……", "...")\
-            .replace("…", "...")\
-            .replace('“', '"')\
-            .replace('”', '"')\
+        text = text.replace("——", "...") \
+            .replace("—", "...") \
+            .replace("……", "...") \
+            .replace("…", "...") \
+            .replace('“', '"') \
+            .replace('”', '"') \
             .replace("\n", "")
         tokens = self.prosody.char_model.tokenizer.tokenize(text)
         text = ''.join(tokens)

vits/vits.py

@@ -1,18 +1,19 @@
 import librosa
-from vits import commons
 import re
 import numpy as np
 import torch
 from torch import no_grad, LongTensor, inference_mode, FloatTensor
-from utils.nlp import sentence_split
+import utils
+from utils.sentence import sentence_split_and_markup
+from vits import commons
 from vits.mel_processing import spectrogram_torch
 from vits.text import text_to_sequence
 from vits.models import SynthesizerTrn
-from utils import utils
+
 
 
 class VITS:
-    def __init__(self, model, config, model_=None, model_type=None, device=torch.device("cpu")):
+    def __init__(self, model, config, additional_model=None, model_type=None, device=torch.device("cpu"),**kwargs):
         self.model_type = model_type
         self.hps_ms = utils.get_hparams_from_file(config)
         self.n_speakers = getattr(self.hps_ms.data, 'n_speakers', 0)
@@ -38,15 +39,15 @@ class VITS:
         self.device = device
 
         # load model
-        self.load_model(model, model_)
+        self.load_model(model, additional_model)
 
-    def load_model(self, model, model_=None):
+    def load_model(self, model, additional_model=None):
         utils.load_checkpoint(model, self.net_g_ms)
         self.net_g_ms.to(self.device)
         if self.model_type == "hubert":
-            self.hubert = model_
+            self.hubert = additional_model
         elif self.model_type == "w2v2":
-            self.emotion_reference = model_
+            self.emotion_reference = additional_model
 
     def get_cleaned_text(self, text, hps, cleaned=False):
         if cleaned:
@@ -165,7 +166,7 @@ class VITS:
 
         tasks = []
         if self.model_type == "vits":
-            sentence_list = sentence_split(text, max, lang, speaker_lang)
+            sentence_list = sentence_split_and_markup(text, max, lang, speaker_lang)
             for sentence in sentence_list:
                 params = self.get_infer_param(text=sentence, speaker_id=speaker_id, length_scale=length,
                                               noise_scale=noise, noise_scale_w=noisew)
@@ -177,7 +178,7 @@ class VITS:
             tasks.append(params)
 
         elif self.model_type == "w2v2":
-            sentence_list = sentence_split(text, max, lang, speaker_lang)
+            sentence_list = sentence_split_and_markup(text, max, lang, speaker_lang)
             for sentence in sentence_list:
                 params = self.get_infer_param(text=sentence, speaker_id=speaker_id, length_scale=length,
                                               noise_scale=noise, noise_scale_w=noisew, emotion=emotion)

voice.py

@@ -8,7 +8,7 @@ import config
 import soundfile as sf
 from io import BytesIO
 from graiax import silkcoder
-from utils import utils
+import utils
 from logger import logger
 
 
@@ -16,7 +16,7 @@ from logger import logger
 
 
 class TTS:
-    def __init__(self, voice_obj, voice_speakers, w2v2_emotion_count=0, device=torch.device("cpu")):
+    def __init__(self, voice_obj, voice_speakers, **kwargs):
         self._voice_obj = voice_obj
         self._voice_speakers = voice_speakers
         self._strength_dict = {"x-weak": 0.25, "weak": 0.5, "Medium": 0.75, "Strong": 1, "x-strong": 1.25}
@@ -24,14 +24,14 @@ class TTS:
         self._vits_speakers_count = len(self._voice_speakers["VITS"])
         self._hubert_speakers_count = len(self._voice_speakers["HUBERT-VITS"])
         self._w2v2_speakers_count = len(self._voice_speakers["W2V2-VITS"])
-        self._w2v2_emotion_count = w2v2_emotion_count
+        self._w2v2_emotion_count = kwargs.get("w2v2_emotion_count", 0)
         self._bert_vits2_speakers_count = len(self._voice_speakers["BERT-VITS2"])
         self.dem = None
 
         # Initialization information
         self.logger = logger
         self.logger.info(f"torch:{torch.__version__} cuda_available:{torch.cuda.is_available()}")
-        self.logger.info(f'device:{device} device.type:{device.type}')
+        self.logger.info(f'device:{kwargs.get("device")} device.type:{kwargs.get("device").type}')
 
         if getattr(config, "DIMENSIONAL_EMOTION_MODEL", None) != None:
             try:
@@ -45,7 +45,8 @@ class TTS:
         if self._vits_speakers_count != 0: self.logger.info(f"[VITS] {self._vits_speakers_count} speakers")
         if self._hubert_speakers_count != 0: self.logger.info(f"[hubert] {self._hubert_speakers_count} speakers")
         if self._w2v2_speakers_count != 0: self.logger.info(f"[w2v2] {self._w2v2_speakers_count} speakers")
-        if self._bert_vits2_speakers_count != 0: self.logger.info(f"[Bert-VITS2] {self._bert_vits2_speakers_count} speakers")
+        if self._bert_vits2_speakers_count != 0: self.logger.info(
+            f"[Bert-VITS2] {self._bert_vits2_speakers_count} speakers")
         self.logger.info(f"{self._speakers_count} speakers in total.")
         if self._speakers_count == 0:
             self.logger.warning(f"No model was loaded.")
@@ -196,84 +197,73 @@ class TTS:
 
         return voice_tasks, format
 
-    def create_ssml_infer_task(self, voice_tasks, format, fname):
+    def create_ssml_infer_task(self, tasks, format):
         audios = []
-        for voice in voice_tasks:
-            if voice.get("break"):
-                audios.append(np.zeros(int(voice.get("break") * 22050), dtype=np.int16))
+        for task in tasks:
+            if task.get("break"):
+                audios.append(np.zeros(int(task.get("break") * 22050), dtype=np.int16))
             else:
-                model = voice.get("model").upper()
+                model = task.get("model").upper()
                 if model != "VITS" and model != "W2V2-VITS" and model != "EMOTION-VITS":
-                    raise ValueError(f"Unsupported model: {voice.get('model')}")
-                voice_obj = self._voice_obj[model][voice.get("id")][1]
-                voice["id"] = self._voice_obj[model][voice.get("id")][0]
-                audio = voice_obj.get_audio(voice)
+                    raise ValueError(f"Unsupported model: {task.get('model')}")
+                voice_obj = self._voice_obj[model][task.get("id")][1]
+                task["id"] = self._voice_obj[model][task.get("id")][0]
+                audio = voice_obj.get_audio(task)
                 audios.append(audio)
 
         audio = np.concatenate(audios, axis=0)
         encoded_audio = self.encode(voice_obj.hps_ms.data.sampling_rate, audio, format)
-        if getattr(config, "SAVE_AUDIO", False):
-            path = f"{config.CACHE_PATH}/{fname}"
-            utils.save_audio(encoded_audio.getvalue(), path)
         return encoded_audio
 
-    def vits_infer(self, voice, fname):
-        format = voice.get("format", "wav")
-        voice_obj = self._voice_obj["VITS"][voice.get("id")][1]
-        voice["id"] = self._voice_obj["VITS"][voice.get("id")][0]
+    def vits_infer(self, task):
+        format = task.get("format", "wav")
+        voice_obj = self._voice_obj["VITS"][task.get("id")][1]
+        real_id = self._voice_obj["VITS"][task.get("id")][0]
+        task["id"] = real_id  # Change to real id
         sampling_rate = voice_obj.hps_ms.data.sampling_rate
-        audio = voice_obj.get_audio(voice, auto_break=True)
+        audio = voice_obj.get_audio(task, auto_break=True)
         encoded_audio = self.encode(sampling_rate, audio, format)
-        if getattr(config, "SAVE_AUDIO", False):
-            path = f"{config.CACHE_PATH}/{fname}"
-            utils.save_audio(encoded_audio.getvalue(), path)
         return encoded_audio
 
-    def stream_vits_infer(self, voice, fname):
-        format = voice.get("format", "wav")
-        voice_obj = self._voice_obj["VITS"][voice.get("id")][1]
-        voice["id"] = self._voice_obj["VITS"][voice.get("id")][0]
+    def stream_vits_infer(self, task, fname=None):
+        format = task.get("format", "wav")
+        voice_obj = self._voice_obj["VITS"][task.get("id")][1]
+        task["id"] = self._voice_obj["VITS"][task.get("id")][0]
         sampling_rate = voice_obj.hps_ms.data.sampling_rate
-        genertator = voice_obj.get_stream_audio(voice, auto_break=True)
-        audio = BytesIO()
+        genertator = voice_obj.get_stream_audio(task, auto_break=True)
+        # audio = BytesIO()
         for chunk in genertator:
             encoded_audio = self.encode(sampling_rate, chunk, format)
             for encoded_audio_chunk in self.generate_audio_chunks(encoded_audio):
                 yield encoded_audio_chunk
-            if getattr(config, "SAVE_AUDIO", False):
-                audio.write(encoded_audio.getvalue())
-        if getattr(config, "SAVE_AUDIO", False):
-            path = f"{config.CACHE_PATH}/{fname}"
-            utils.save_audio(audio.getvalue(), path)
-
-    def hubert_vits_infer(self, voice, fname):
-        format = voice.get("format", "wav")
-        voice_obj = self._voice_obj["HUBERT-VITS"][voice.get("id")][1]
-        voice["id"] = self._voice_obj["HUBERT-VITS"][voice.get("id")][0]
+        #     if getattr(config, "SAVE_AUDIO", False):
+        #         audio.write(encoded_audio.getvalue())
+        # if getattr(config, "SAVE_AUDIO", False):
+        #     path = f"{config.CACHE_PATH}/{fname}"
+        #     utils.save_audio(audio.getvalue(), path)
+
+    def hubert_vits_infer(self, task):
+        format = task.get("format", "wav")
+        voice_obj = self._voice_obj["HUBERT-VITS"][task.get("id")][1]
+        task["id"] = self._voice_obj["HUBERT-VITS"][task.get("id")][0]
         sampling_rate = voice_obj.hps_ms.data.sampling_rate
-        audio = voice_obj.get_audio(voice)
+        audio = voice_obj.get_audio(task)
         encoded_audio = self.encode(sampling_rate, audio, format)
-        if getattr(config, "SAVE_AUDIO", False):
-            path = f"{config.CACHE_PATH}/{fname}"
-            utils.save_audio(encoded_audio.getvalue(), path)
         return encoded_audio
 
-    def w2v2_vits_infer(self, voice, fname):
-        format = voice.get("format", "wav")
-        voice_obj = self._voice_obj["W2V2-VITS"][voice.get("id")][1]
-        voice["id"] = self._voice_obj["W2V2-VITS"][voice.get("id")][0]
+    def w2v2_vits_infer(self, task):
+        format = task.get("format", "wav")
+        voice_obj = self._voice_obj["W2V2-VITS"][task.get("id")][1]
+        task["id"] = self._voice_obj["W2V2-VITS"][task.get("id")][0]
         sampling_rate = voice_obj.hps_ms.data.sampling_rate
-        audio = voice_obj.get_audio(voice, auto_break=True)
+        audio = voice_obj.get_audio(task, auto_break=True)
         encoded_audio = self.encode(sampling_rate, audio, format)
-        if getattr(config, "SAVE_AUDIO", False):
-            path = f"{config.CACHE_PATH}/{fname}"
-            utils.save_audio(encoded_audio.getvalue(), path)
         return encoded_audio
 
-    def vits_voice_conversion(self, voice, fname):
-        original_id = voice.get("original_id")
-        target_id = voice.get("target_id")
-        format = voice.get("format")
+    def vits_voice_conversion(self, task):
+        original_id = task.get("original_id")
+        target_id = task.get("target_id")
+        format = task.get("format")
 
         original_id_obj = int(self._voice_obj["VITS"][original_id][2])
         target_id_obj = int(self._voice_obj["VITS"][target_id][2])
@@ -281,17 +271,14 @@ class TTS:
         if original_id_obj != target_id_obj:
             raise ValueError(f"speakers are in diffrent VITS Model")
 
-        voice["original_id"] = int(self._voice_obj["VITS"][original_id][0])
-        voice["target_id"] = int(self._voice_obj["VITS"][target_id][0])
+        task["original_id"] = int(self._voice_obj["VITS"][original_id][0])
+        task["target_id"] = int(self._voice_obj["VITS"][target_id][0])
 
         voice_obj = self._voice_obj["VITS"][original_id][1]
         sampling_rate = voice_obj.hps_ms.data.sampling_rate
 
-        audio = voice_obj.voice_conversion(voice)
+        audio = voice_obj.voice_conversion(task)
         encoded_audio = self.encode(sampling_rate, audio, format)
-        if getattr(config, "SAVE_AUDIO", False):
-            path = f"{config.CACHE_PATH}/{fname}"
-            utils.save_audio(encoded_audio.getvalue(), path)
         return encoded_audio
 
     def get_dimensional_emotion_npy(self, audio):
@@ -305,14 +292,11 @@ class TTS:
 
         return emotion_npy
 
-    def bert_vits2_infer(self, voice, fname):
-        format = voice.get("format", "wav")
-        voice_obj = self._voice_obj["BERT-VITS2"][voice.get("id")][1]
-        voice["id"] = self._voice_obj["BERT-VITS2"][voice.get("id")][0]
+    def bert_vits2_infer(self, task):
+        format = task.get("format", "wav")
+        voice_obj = self._voice_obj["BERT-VITS2"][task.get("id")][1]
+        task["id"] = self._voice_obj["BERT-VITS2"][task.get("id")][0]
         sampling_rate = voice_obj.hps_ms.data.sampling_rate
-        audio = voice_obj.get_audio(voice, auto_break=True)
+        audio = voice_obj.get_audio(task, auto_break=True)
         encoded_audio = self.encode(sampling_rate, audio, format)
-        if getattr(config, "SAVE_AUDIO", False):
-            path = f"{config.CACHE_PATH}/{fname}"
-            utils.save_audio(encoded_audio.getvalue(), path)
         return encoded_audio