update to current version

InferenceInterfaces/ControllableInterface.py

@@ -2,8 +2,8 @@ import os
 
 import torch
 
-from Architectures.ControllabilityGAN.GAN import GanWrapper
 from InferenceInterfaces.ToucanTTSInterface import ToucanTTSInterface
+from Modules.ControllabilityGAN.GAN import GanWrapper
 from Utility.storage_config import MODELS_DIR
 
 
@@ -16,14 +16,18 @@ class ControllableInterface:
             os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
             os.environ["CUDA_VISIBLE_DEVICES"] = f"{gpu_id}"
         self.device = "cuda" if gpu_id != "cpu" else "cpu"
-        self.model = ToucanTTSInterface(device=self.device, tts_model_path="Meta", language="eng")
+        self.model = ToucanTTSInterface(device=self.device, tts_model_path="Meta")
         self.wgan = GanWrapper(os.path.join(MODELS_DIR, "Embedding", "embedding_gan.pt"), device=self.device)
         self.generated_speaker_embeds = list()
         self.available_artificial_voices = available_artificial_voices
+        self.current_language = ""
+        self.current_accent = ""
 
     def read(self,
              prompt,
-             audio,
+             reference_audio,
+             language,
+             accent,
              voice_seed,
              prosody_creativity,
              duration_scaling_factor,
@@ -38,7 +42,15 @@ class ControllableInterface:
              emb_slider_6,
              loudness_in_db
              ):
-        if audio is None:
+        if self.current_language != language:
+            self.model.set_phonemizer_language(language)
+            print(f"switched phonemizer language to {language}")
+            self.current_language = language
+        if self.current_accent != accent:
+            self.model.set_accent_language(accent)
+            print(f"switched accent language to {accent}")
+            self.current_accent = accent
+        if reference_audio is None:
             self.wgan.set_latent(voice_seed)
             controllability_vector = torch.tensor([emb_slider_1,
                                                    emb_slider_2,
@@ -49,13 +61,46 @@ class ControllableInterface:
             embedding = self.wgan.modify_embed(controllability_vector)
             self.model.set_utterance_embedding(embedding=embedding)
         else:
-            self.model.set_utterance_embedding(path_to_reference_audio=audio)
+            self.model.set_utterance_embedding(reference_audio)
 
         phones = self.model.text2phone.get_phone_string(prompt)
         if len(phones) > 1800:
-           prompt = "Your input was too long. Please try either a shorter text or split it into several parts."
+            if language == "deu":
+                prompt = "Deine Eingabe war zu lang. Bitte versuche es entweder mit einem kürzeren Text oder teile ihn in mehrere Teile auf."
+            elif language == "ell":
+                prompt = "Η εισήγησή σας ήταν πολύ μεγάλη. Παρακαλώ δοκιμάστε είτε ένα μικρότερο κείμενο είτε χωρίστε το σε διάφορα μέρη."
+            elif language == "spa":
+                prompt = "Su entrada es demasiado larga. Por favor, intente un texto más corto o divídalo en varias partes."
+            elif language == "fin":
+                prompt = "Vastauksesi oli liian pitkä. Kokeile joko lyhyempää tekstiä tai jaa se useampaan osaan."
+            elif language == "rus":
+                prompt = "Ваш текст слишком длинный. Пожалуйста, попробуйте либо сократить текст, либо разделить его на несколько частей."
+            elif language == "hun":
+                prompt = "Túl hosszú volt a bevitele. Kérjük, próbáljon meg rövidebb szöveget írni, vagy ossza több részre."
+            elif language == "nld":
+                prompt = "Uw input was te lang. Probeer een kortere tekst of splits het in verschillende delen."
+            elif language == "fra":
+                prompt = "Votre saisie était trop longue. Veuillez essayer un texte plus court ou le diviser en plusieurs parties."
+            elif language == 'pol':
+                prompt = "Twój wpis był zbyt długi. Spróbuj skrócić tekst lub podzielić go na kilka części."
+            elif language == 'por':
+                prompt = "O seu contributo foi demasiado longo. Por favor, tente um texto mais curto ou divida-o em várias partes."
+            elif language == 'ita':
+                prompt = "Il tuo input era troppo lungo. Per favore, prova un testo più corto o dividilo in più parti."
+            elif language == 'cmn':
+                prompt = "你的输入太长了。请尝试使用较短的文本或将其拆分为多个部分。"
+            elif language == 'vie':
+                prompt = "Đầu vào của bạn quá dài. Vui lòng thử một văn bản ngắn hơn hoặc chia nó thành nhiều phần."
+            else:
+                prompt = "Your input was too long. Please try either a shorter text or split it into several parts."
+                if self.current_language != "eng":
+                    self.model.set_phonemizer_language("eng")
+                    self.current_language = "eng"
+                if self.current_accent != "eng":
+                    self.model.set_accent_language("eng")
+                    self.current_accent = "eng"
 
-        print(prompt)
+        print(prompt + "\n\n")
         wav, sr, fig = self.model(prompt,
                                   input_is_phones=False,
                                   duration_scaling_factor=duration_scaling_factor,

InferenceInterfaces/ToucanTTSInterface.py

@@ -10,8 +10,8 @@ import torch
 from speechbrain.pretrained import EncoderClassifier
 from torchaudio.transforms import Resample
 
-from Architectures.ToucanTTS.InferenceToucanTTS import ToucanTTS
-from Architectures.Vocoder.HiFiGAN_Generator import HiFiGAN
+from Modules.ToucanTTS.InferenceToucanTTS import ToucanTTS
+from Modules.Vocoder.HiFiGAN_Generator import HiFiGAN
 from Preprocessing.AudioPreprocessor import AudioPreprocessor
 from Preprocessing.TextFrontend import ArticulatoryCombinedTextFrontend
 from Preprocessing.TextFrontend import get_language_id
@@ -109,7 +109,7 @@ class ToucanTTSInterface(torch.nn.Module):
         self.text2phone = ArticulatoryCombinedTextFrontend(language=lang_id, add_silence_to_end=True, device=self.device)
 
     def set_accent_language(self, lang_id):
-        if lang_id in ['ajp', 'ajt', 'lak', 'lno', 'nul', 'pii', 'plj', 'slq', 'smd', 'snb', 'tpw', 'wya', 'zua', 'en-us', 'en-sc', 'fr-be', 'fr-sw', 'pt-br', 'spa-lat', 'vi-ctr', 'vi-so']:
+        if lang_id in {'ajp', 'ajt', 'lak', 'lno', 'nul', 'pii', 'plj', 'slq', 'smd', 'snb', 'tpw', 'wya', 'zua', 'en-us', 'en-sc', 'fr-be', 'fr-sw', 'pt-br', 'spa-lat', 'vi-ctr', 'vi-so'}:
             if lang_id == 'vi-so' or lang_id == 'vi-ctr':
                 lang_id = 'vie'
             elif lang_id == 'spa-lat':
@@ -121,7 +121,7 @@ class ToucanTTSInterface(torch.nn.Module):
             elif lang_id == 'en-sc' or lang_id == 'en-us':
                 lang_id = 'eng'
             else:
-                # no clue where these others are even coming from, they are not in ISO 639-2
+                # no clue where these others are even coming from, they are not in ISO 639-3
                 lang_id = 'eng'
 
         self.lang_id = get_language_id(lang_id).to(self.device)
@@ -139,7 +139,7 @@ class ToucanTTSInterface(torch.nn.Module):
                 input_is_phones=False,
                 return_plot_as_filepath=False,
                 loudness_in_db=-24.0,
-                prosody_creativity=0.5):
+                prosody_creativity=0.1):
         """
         duration_scaling_factor: reasonable values are 0.8 < scale < 1.2.
                                      1.0 means no scaling happens, higher values increase durations for the whole
@@ -241,7 +241,7 @@ class ToucanTTSInterface(torch.nn.Module):
                      dur_list=None,
                      pitch_list=None,
                      energy_list=None,
-                     prosody_creativity=0.5):
+                     prosody_creativity=0.1):
         """
         Args:
             silent: Whether to be verbose about the process
@@ -299,7 +299,7 @@ class ToucanTTSInterface(torch.nn.Module):
                    pitch_variance_scale=1.0,
                    energy_variance_scale=1.0,
                    blocking=False,
-                   prosody_creativity=0.5):
+                   prosody_creativity=0.1):
         if text.strip() == "":
             return
         wav, sr = self(text,

InferenceInterfaces/UtteranceCloner.py

@@ -4,11 +4,11 @@ import numpy
 import soundfile as sf
 import torch
 
-from Architectures.Aligner.Aligner import Aligner
-from Architectures.ToucanTTS.DurationCalculator import DurationCalculator
-from Architectures.ToucanTTS.EnergyCalculator import EnergyCalculator
-from Architectures.ToucanTTS.PitchCalculator import Parselmouth
 from InferenceInterfaces.ToucanTTSInterface import ToucanTTSInterface
+from Modules.Aligner.Aligner import Aligner
+from Modules.ToucanTTS.DurationCalculator import DurationCalculator
+from Modules.ToucanTTS.EnergyCalculator import EnergyCalculator
+from Modules.ToucanTTS.PitchCalculator import Parselmouth
 from Preprocessing.AudioPreprocessor import AudioPreprocessor
 from Preprocessing.TextFrontend import ArticulatoryCombinedTextFrontend
 from Preprocessing.articulatory_features import get_feature_to_index_lookup

Models/ToucanTTS_Meta/best.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3f1f562f9473f227b4425938c80dec1808d0cd3a54fd3629b327613dae3be694
-size 112081651

Models/Vocoder/best.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:68a4db7d7d96a554eab75c5d8b79267760d7d4c7af65504947ab807ab18d680b
-size 56113099

{Architectures → Modules}/Aligner/autoaligner_train_loop.py

@@ -8,8 +8,8 @@ from torch.optim import RAdam
 from torch.utils.data.dataloader import DataLoader
 from tqdm import tqdm
 
-from Architectures.Aligner.Aligner import Aligner
-from Architectures.Aligner.Reconstructor import Reconstructor
+from Modules.Aligner.Aligner import Aligner
+from Modules.Aligner.Reconstructor import Reconstructor
 from Preprocessing.AudioPreprocessor import AudioPreprocessor
 from Preprocessing.EnCodecAudioPreprocessor import CodecAudioPreprocessor
 

{Architectures → Modules}/ControllabilityGAN/GAN.py

@@ -1,6 +1,6 @@
 import torch
 
-from Architectures.ControllabilityGAN.wgan.init_wgan import create_wgan
+from Modules.ControllabilityGAN.wgan.init_wgan import create_wgan
 
 
 class GanWrapper:

{Architectures → Modules}/ControllabilityGAN/wgan/init_wgan.py

@@ -1,7 +1,7 @@
 import torch
 
-from Architectures.ControllabilityGAN.wgan.resnet_init import init_resnet
-from Architectures.ControllabilityGAN.wgan.wgan_qc import WassersteinGanQuadraticCost
+from Modules.ControllabilityGAN.wgan.resnet_init import init_resnet
+from Modules.ControllabilityGAN.wgan.wgan_qc import WassersteinGanQuadraticCost
 
 
 def create_wgan(parameters, device, optimizer='adam'):

{Architectures → Modules}/ControllabilityGAN/wgan/resnet_init.py

@@ -1,7 +1,7 @@
-from Architectures.ControllabilityGAN.wgan.init_weights import weights_init_D
-from Architectures.ControllabilityGAN.wgan.init_weights import weights_init_G
-from Architectures.ControllabilityGAN.wgan.resnet_1 import ResNet_D
-from Architectures.ControllabilityGAN.wgan.resnet_1 import ResNet_G
+from Modules.ControllabilityGAN.wgan.init_weights import weights_init_D
+from Modules.ControllabilityGAN.wgan.init_weights import weights_init_G
+from Modules.ControllabilityGAN.wgan.resnet_1 import ResNet_D
+from Modules.ControllabilityGAN.wgan.resnet_1 import ResNet_G
 
 
 def init_resnet(parameters):

{Architectures → Modules}/EmbeddingModel/GST.py

@@ -3,7 +3,7 @@
 
 import torch
 
-from Architectures.GeneralLayers.Attention import MultiHeadedAttention as BaseMultiHeadedAttention
+from Modules.GeneralLayers.Attention import MultiHeadedAttention as BaseMultiHeadedAttention
 
 
 class GSTStyleEncoder(torch.nn.Module):

{Architectures → Modules}/EmbeddingModel/StyleEmbedding.py

@@ -1,7 +1,7 @@
 import torch
 
-from Architectures.EmbeddingModel.GST import GSTStyleEncoder
-from Architectures.EmbeddingModel.StyleTTSEncoder import StyleEncoder as StyleTTSEncoder
+from Modules.EmbeddingModel.GST import GSTStyleEncoder
+from Modules.EmbeddingModel.StyleTTSEncoder import StyleEncoder as StyleTTSEncoder
 
 
 class StyleEmbedding(torch.nn.Module):

{Architectures → Modules}/GeneralLayers/ConditionalLayerNorm.py

@@ -112,7 +112,6 @@ class AdaIN1d(nn.Module):
         self.fc = nn.Linear(style_dim, num_features * 2)
 
     def forward(self, x, s):
-        s = torch.nn.functional.normalize(s)
         h = self.fc(s)
         h = h.view(h.size(0), h.size(1), 1)
         gamma, beta = torch.chunk(h, chunks=2, dim=1)

{Architectures → Modules}/GeneralLayers/Conformer.py

@@ -4,16 +4,16 @@ Taken from ESPNet, but heavily modified
 
 import torch
 
-from Architectures.GeneralLayers.Attention import RelPositionMultiHeadedAttention
-from Architectures.GeneralLayers.ConditionalLayerNorm import AdaIN1d
-from Architectures.GeneralLayers.ConditionalLayerNorm import ConditionalLayerNorm
-from Architectures.GeneralLayers.Convolution import ConvolutionModule
-from Architectures.GeneralLayers.EncoderLayer import EncoderLayer
-from Architectures.GeneralLayers.LayerNorm import LayerNorm
-from Architectures.GeneralLayers.MultiLayeredConv1d import MultiLayeredConv1d
-from Architectures.GeneralLayers.MultiSequential import repeat
-from Architectures.GeneralLayers.PositionalEncoding import RelPositionalEncoding
-from Architectures.GeneralLayers.Swish import Swish
+from Modules.GeneralLayers.Attention import RelPositionMultiHeadedAttention
+from Modules.GeneralLayers.ConditionalLayerNorm import AdaIN1d
+from Modules.GeneralLayers.ConditionalLayerNorm import ConditionalLayerNorm
+from Modules.GeneralLayers.Convolution import ConvolutionModule
+from Modules.GeneralLayers.EncoderLayer import EncoderLayer
+from Modules.GeneralLayers.LayerNorm import LayerNorm
+from Modules.GeneralLayers.MultiLayeredConv1d import MultiLayeredConv1d
+from Modules.GeneralLayers.MultiSequential import repeat
+from Modules.GeneralLayers.PositionalEncoding import RelPositionalEncoding
+from Modules.GeneralLayers.Swish import Swish
 from Utility.utils import integrate_with_utt_embed
 
 
@@ -88,6 +88,8 @@ class Conformer(torch.nn.Module):
                 self.language_embedding_projection = lambda x: x
             else:
                 self.language_embedding_projection = torch.nn.Linear(lang_emb_size, attention_dim)
+            self.language_emb_norm = LayerNorm(attention_dim)
+
         # self-attention module definition
         encoder_selfattn_layer = RelPositionMultiHeadedAttention
         encoder_selfattn_layer_args = (attention_heads, attention_dim, attention_dropout_rate, zero_triu)
@@ -130,6 +132,7 @@ class Conformer(torch.nn.Module):
         if lang_ids is not None:
             lang_embs = self.language_embedding(lang_ids)
             projected_lang_embs = self.language_embedding_projection(lang_embs).unsqueeze(-1).transpose(1, 2)
+            projected_lang_embs = self.language_emb_norm(projected_lang_embs)
             xs = xs + projected_lang_embs  # offset phoneme representation by language specific offset
 
         xs = self.pos_enc(xs)
@@ -139,21 +142,28 @@ class Conformer(torch.nn.Module):
                 if isinstance(xs, tuple):
                     x, pos_emb = xs[0], xs[1]
                     if self.conformer_type != "encoder":
-                        x = integrate_with_utt_embed(hs=x, utt_embeddings=utterance_embedding, projection=self.decoder_embedding_projections[encoder_index], embedding_training=self.use_conditional_layernorm_embedding_integration)
+                        x = integrate_with_utt_embed(hs=x,
+                                                     utt_embeddings=utterance_embedding,
+                                                     projection=self.decoder_embedding_projections[encoder_index],
+                                                     embedding_training=self.use_conditional_layernorm_embedding_integration)
                     xs = (x, pos_emb)
                 else:
                     if self.conformer_type != "encoder":
-                        xs = integrate_with_utt_embed(hs=xs, utt_embeddings=utterance_embedding, projection=self.decoder_embedding_projections[encoder_index], embedding_training=self.use_conditional_layernorm_embedding_integration)
+                        xs = integrate_with_utt_embed(hs=xs,
+                                                      utt_embeddings=utterance_embedding,
+                                                      projection=self.decoder_embedding_projections[encoder_index],
+                                                      embedding_training=self.use_conditional_layernorm_embedding_integration)
             xs, masks = encoder(xs, masks)
 
         if isinstance(xs, tuple):
             xs = xs[0]
 
-        if self.use_output_norm and not (self.utt_embed and self.conformer_type == "encoder"):
-            xs = self.output_norm(xs)
-
         if self.utt_embed and self.conformer_type == "encoder":
-            xs = integrate_with_utt_embed(hs=xs, utt_embeddings=utterance_embedding,
-                                          projection=self.encoder_embedding_projection, embedding_training=self.use_conditional_layernorm_embedding_integration)
+            xs = integrate_with_utt_embed(hs=xs,
+                                          utt_embeddings=utterance_embedding,
+                                          projection=self.encoder_embedding_projection,
+                                          embedding_training=self.use_conditional_layernorm_embedding_integration)
+        elif self.use_output_norm:
+            xs = self.output_norm(xs)
 
         return xs, masks

{Architectures → Modules}/GeneralLayers/DurationPredictor.py

@@ -5,9 +5,9 @@
 
 import torch
 
-from Architectures.GeneralLayers.ConditionalLayerNorm import AdaIN1d
-from Architectures.GeneralLayers.ConditionalLayerNorm import ConditionalLayerNorm
-from Architectures.GeneralLayers.LayerNorm import LayerNorm
+from Modules.GeneralLayers.ConditionalLayerNorm import AdaIN1d
+from Modules.GeneralLayers.ConditionalLayerNorm import ConditionalLayerNorm
+from Modules.GeneralLayers.LayerNorm import LayerNorm
 from Utility.utils import integrate_with_utt_embed
 
 

{Architectures → Modules}/GeneralLayers/EncoderLayer.py

@@ -7,7 +7,7 @@
 import torch
 from torch import nn
 
-from Architectures.GeneralLayers.LayerNorm import LayerNorm
+from Modules.GeneralLayers.LayerNorm import LayerNorm
 
 
 class EncoderLayer(nn.Module):

{Architectures → Modules}/GeneralLayers/VariancePredictor.py

@@ -6,9 +6,9 @@ from abc import ABC
 
 import torch
 
-from Architectures.GeneralLayers.ConditionalLayerNorm import AdaIN1d
-from Architectures.GeneralLayers.ConditionalLayerNorm import ConditionalLayerNorm
-from Architectures.GeneralLayers.LayerNorm import LayerNorm
+from Modules.GeneralLayers.ConditionalLayerNorm import AdaIN1d
+from Modules.GeneralLayers.ConditionalLayerNorm import ConditionalLayerNorm
+from Modules.GeneralLayers.LayerNorm import LayerNorm
 from Utility.utils import integrate_with_utt_embed
 
 

{Architectures → Modules}/ToucanTTS/CodecRefinementTransformer.py

@@ -1,6 +1,6 @@
 import torch
 
-from Architectures.GeneralLayers.Conformer import Conformer
+from Modules.GeneralLayers.Conformer import Conformer
 
 
 class CodecRefinementTransformer(torch.nn.Module):
@@ -151,7 +151,7 @@ def one_hot_sequence_to_token_sequence(batch_of_indexes_one_hot_per_codebook):
 
 
 if __name__ == '__main__':
-    from Architectures.ToucanTTS.ToucanTTS import ToucanTTS
+    from Modules.ToucanTTS.ToucanTTS import ToucanTTS
     from Utility.utils import make_pad_mask
 
     # prepare dummy inputs

{Architectures → Modules}/ToucanTTS/EnergyCalculator.py

@@ -5,7 +5,7 @@
 import torch
 import torch.nn.functional as F
 
-from Architectures.GeneralLayers.STFT import STFT
+from Modules.GeneralLayers.STFT import STFT
 from Utility.utils import pad_list