add ddsp-svc

DDSP-SVC/.gitignore

@@ -0,0 +1,10 @@
+__pycache__/
+
+venv/
+results/
+configs/
+!configs/combsub.yaml
+!configs/combsub-old.yaml
+!configs/sins.yaml
+!configs/diffusion.yaml
+cache/

DDSP-SVC/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 yxlllc
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

DDSP-SVC/data/train/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!audio

DDSP-SVC/data/train/audio/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

DDSP-SVC/data/val/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!audio

DDSP-SVC/data/val/audio/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

DDSP-SVC/data_loaders.py

@@ -0,0 +1,244 @@
+import os
+import random
+import numpy as np
+import librosa
+import torch
+import random
+from tqdm import tqdm
+from torch.utils.data import Dataset
+
+def traverse_dir(
+        root_dir,
+        extension,
+        amount=None,
+        str_include=None,
+        str_exclude=None,
+        is_pure=False,
+        is_sort=False,
+        is_ext=True):
+
+    file_list = []
+    cnt = 0
+    for root, _, files in os.walk(root_dir):
+        for file in files:
+            if file.endswith(extension):
+                # path
+                mix_path = os.path.join(root, file)
+                pure_path = mix_path[len(root_dir)+1:] if is_pure else mix_path
+
+                # amount
+                if (amount is not None) and (cnt == amount):
+                    if is_sort:
+                        file_list.sort()
+                    return file_list
+                
+                # check string
+                if (str_include is not None) and (str_include not in pure_path):
+                    continue
+                if (str_exclude is not None) and (str_exclude in pure_path):
+                    continue
+                
+                if not is_ext:
+                    ext = pure_path.split('.')[-1]
+                    pure_path = pure_path[:-(len(ext)+1)]
+                file_list.append(pure_path)
+                cnt += 1
+    if is_sort:
+        file_list.sort()
+    return file_list
+
+
+def get_data_loaders(args, whole_audio=False):
+    data_train = AudioDataset(
+        args.data.train_path,
+        waveform_sec=args.data.duration,
+        hop_size=args.data.block_size,
+        sample_rate=args.data.sampling_rate,
+        load_all_data=args.train.cache_all_data,
+        whole_audio=whole_audio,
+        n_spk=args.model.n_spk,
+        device=args.train.cache_device,
+        fp16=args.train.cache_fp16,
+        use_aug=True)
+    loader_train = torch.utils.data.DataLoader(
+        data_train ,
+        batch_size=args.train.batch_size if not whole_audio else 1,
+        shuffle=True,
+        num_workers=args.train.num_workers if args.train.cache_device=='cpu' else 0,
+        persistent_workers=(args.train.num_workers > 0) if args.train.cache_device=='cpu' else False,
+        pin_memory=True if args.train.cache_device=='cpu' else False
+    )
+    data_valid = AudioDataset(
+        args.data.valid_path,
+        waveform_sec=args.data.duration,
+        hop_size=args.data.block_size,
+        sample_rate=args.data.sampling_rate,
+        load_all_data=args.train.cache_all_data,
+        whole_audio=True,
+        n_spk=args.model.n_spk)
+    loader_valid = torch.utils.data.DataLoader(
+        data_valid,
+        batch_size=1,
+        shuffle=False,
+        num_workers=0,
+        pin_memory=True
+    )
+    return loader_train, loader_valid 
+
+
+class AudioDataset(Dataset):
+    def __init__(
+        self,
+        path_root,
+        waveform_sec,
+        hop_size,
+        sample_rate,
+        load_all_data=True,
+        whole_audio=False,
+        n_spk=1,
+        device = 'cpu',
+        fp16 = False,
+        use_aug = False
+    ):
+        super().__init__()
+        
+        self.waveform_sec = waveform_sec
+        self.sample_rate = sample_rate
+        self.hop_size = hop_size
+        self.path_root = path_root
+        self.paths = traverse_dir(
+            os.path.join(path_root, 'audio'),
+            extension='wav',
+            is_pure=True,
+            is_sort=True,
+            is_ext=False
+        )
+        self.whole_audio = whole_audio
+        self.use_aug = use_aug
+        self.data_buffer={}
+        if load_all_data:
+            print('Load all the data from :', path_root)
+        else:
+            print('Load the f0, volume data from :', path_root)
+        for name in tqdm(self.paths, total=len(self.paths)):
+            path_audio = os.path.join(self.path_root, 'audio', name) + '.wav'
+            duration = librosa.get_duration(filename = path_audio, sr = self.sample_rate)
+            
+            path_f0 = os.path.join(self.path_root, 'f0', name) + '.npy'
+            f0 = np.load(path_f0)
+            f0 = torch.from_numpy(f0).float().unsqueeze(-1).to(device)
+                
+            path_volume = os.path.join(self.path_root, 'volume', name) + '.npy'
+            volume = np.load(path_volume)
+            volume = torch.from_numpy(volume).float().unsqueeze(-1).to(device)
+            
+            if n_spk is not None and n_spk > 1:
+                spk_id = int(os.path.dirname(name)) if str.isdigit(os.path.dirname(name)) else 0
+                if spk_id < 1 or spk_id > n_spk:
+                    raise ValueError(' [x] Muiti-speaker traing error : spk_id must be a positive integer from 1 to n_spk ')
+            else:
+                spk_id = 1
+            spk_id = torch.LongTensor(np.array([spk_id])).to(device)
+
+            if load_all_data:
+                audio, sr = librosa.load(path_audio, sr=self.sample_rate)
+                if len(audio.shape) > 1:
+                    audio = librosa.to_mono(audio)
+                audio = torch.from_numpy(audio).to(device)
+                
+                path_units = os.path.join(self.path_root, 'units', name) + '.npy'
+                units = np.load(path_units)
+                units = torch.from_numpy(units).to(device)
+                
+                if fp16:
+                    audio = audio.half()
+                    units = units.half()
+                    
+                self.data_buffer[name] = {
+                        'duration': duration,
+                        'audio': audio,
+                        'units': units,
+                        'f0': f0,
+                        'volume': volume,
+                        'spk_id': spk_id
+                        }
+            else:
+                self.data_buffer[name] = {
+                        'duration': duration,
+                        'f0': f0,
+                        'volume': volume,
+                        'spk_id': spk_id
+                        }
+           
+
+    def __getitem__(self, file_idx):
+        name = self.paths[file_idx]
+        data_buffer = self.data_buffer[name]
+        # check duration. if too short, then skip
+        if data_buffer['duration'] < (self.waveform_sec + 0.1):
+            return self.__getitem__( (file_idx + 1) % len(self.paths))
+            
+        # get item
+        return self.get_data(name, data_buffer)
+
+    def get_data(self, name, data_buffer):
+        frame_resolution = self.hop_size / self.sample_rate
+        duration = data_buffer['duration']
+        waveform_sec = duration if self.whole_audio else self.waveform_sec
+        
+        # load audio
+        idx_from = 0 if self.whole_audio else random.uniform(0, duration - waveform_sec - 0.1)
+        start_frame = int(idx_from / frame_resolution)
+        units_frame_len = int(waveform_sec / frame_resolution)
+        audio = data_buffer.get('audio')
+        if audio is None:
+            path_audio = os.path.join(self.path_root, 'audio', name) + '.wav'
+            audio, sr = librosa.load(
+                    path_audio, 
+                    sr = self.sample_rate, 
+                    offset = start_frame * frame_resolution,
+                    duration = waveform_sec)
+            if len(audio.shape) > 1:
+                audio = librosa.to_mono(audio)
+            # clip audio into N seconds
+            audio = audio[ : audio.shape[-1] // self.hop_size * self.hop_size]       
+            audio = torch.from_numpy(audio).float()
+        else:
+            audio = audio[start_frame * self.hop_size : (start_frame + units_frame_len) * self.hop_size]
+        
+        # load units
+        units = data_buffer.get('units')
+        if units is None:
+            units  = os.path.join(self.path_root, 'units', name) + '.npy'
+            units = np.load(units)
+            units = units[start_frame : start_frame + units_frame_len]
+            units = torch.from_numpy(units).float() 
+        else:
+            units = units[start_frame : start_frame + units_frame_len]
+
+        # load f0
+        f0 = data_buffer.get('f0')
+        f0_frames = f0[start_frame : start_frame + units_frame_len]
+        
+        # load volume
+        volume = data_buffer.get('volume')
+        volume_frames = volume[start_frame : start_frame + units_frame_len]
+        
+        # load spk_id
+        spk_id = data_buffer.get('spk_id')
+        
+        # volume augmentation
+        if self.use_aug:
+            max_amp = float(torch.max(torch.abs(audio))) + 1e-5
+            max_shift = min(1, np.log10(1/max_amp))
+            log10_vol_shift = random.uniform(-1, max_shift)
+            audio_aug = audio * (10 ** log10_vol_shift)
+            volume_frames_aug = volume_frames * (10 ** log10_vol_shift)
+        else:
+            audio_aug = audio
+            volume_frames_aug = volume_frames
+        
+        return dict(audio=audio_aug, f0=f0_frames, volume=volume_frames_aug, units=units, spk_id=spk_id, name=name)
+
+    def __len__(self):
+        return len(self.paths)

DDSP-SVC/ddsp/core.py

@@ -0,0 +1,281 @@
+import torch
+import torch.nn as nn
+from torch.nn import functional as F
+
+import math
+import numpy as np
+
+def MaskedAvgPool1d(x, kernel_size):
+    x = x.unsqueeze(1)
+    x = F.pad(x, ((kernel_size - 1) // 2, kernel_size // 2), mode="reflect")
+    mask = ~torch.isnan(x)
+    masked_x = torch.where(mask, x, torch.zeros_like(x))
+    ones_kernel = torch.ones(x.size(1), 1, kernel_size, device=x.device)
+
+    # Perform sum pooling
+    sum_pooled = F.conv1d(
+        masked_x,
+        ones_kernel,
+        stride=1,
+        padding=0,
+        groups=x.size(1),
+    )
+
+    # Count the non-masked (valid) elements in each pooling window
+    valid_count = F.conv1d(
+        mask.float(),
+        ones_kernel,
+        stride=1,
+        padding=0,
+        groups=x.size(1),
+    )
+    valid_count = valid_count.clamp(min=1)  # Avoid division by zero
+
+    # Perform masked average pooling
+    avg_pooled = sum_pooled / valid_count
+
+    return avg_pooled.squeeze(1)
+
+def MedianPool1d(x, kernel_size):
+    x = x.unsqueeze(1)
+    x = F.pad(x, ((kernel_size - 1) // 2, kernel_size // 2), mode="reflect")
+    x = x.squeeze(1)
+    x = x.unfold(1, kernel_size, 1)
+    x, _ = torch.sort(x, dim=-1)
+    return x[:, :, (kernel_size - 1) // 2]
+    
+def get_fft_size(frame_size: int, ir_size: int, power_of_2: bool = True):
+  """Calculate final size for efficient FFT.
+  Args:
+    frame_size: Size of the audio frame.
+    ir_size: Size of the convolving impulse response.
+    power_of_2: Constrain to be a power of 2. If False, allow other 5-smooth
+      numbers. TPU requires power of 2, while GPU is more flexible.
+  Returns:
+    fft_size: Size for efficient FFT.
+  """
+  convolved_frame_size = ir_size + frame_size - 1
+  if power_of_2:
+    # Next power of 2.
+    fft_size = int(2**np.ceil(np.log2(convolved_frame_size)))
+  else:
+    fft_size = convolved_frame_size
+  return fft_size
+
+
+def upsample(signal, factor):
+    signal = signal.permute(0, 2, 1)
+    signal = nn.functional.interpolate(torch.cat((signal,signal[:,:,-1:]),2), size=signal.shape[-1] * factor + 1, mode='linear', align_corners=True)
+    signal = signal[:,:,:-1]
+    return signal.permute(0, 2, 1)
+
+
+def remove_above_fmax(amplitudes, pitch, fmax, level_start=1):
+    n_harm = amplitudes.shape[-1]
+    pitches = pitch * torch.arange(level_start, n_harm + level_start).to(pitch)
+    aa = (pitches < fmax).float() + 1e-7
+    return amplitudes * aa
+
+
+def crop_and_compensate_delay(audio, audio_size, ir_size,
+                              padding = 'same',
+                              delay_compensation = -1):
+  """Crop audio output from convolution to compensate for group delay.
+  Args:
+    audio: Audio after convolution. Tensor of shape [batch, time_steps].
+    audio_size: Initial size of the audio before convolution.
+    ir_size: Size of the convolving impulse response.
+    padding: Either 'valid' or 'same'. For 'same' the final output to be the
+      same size as the input audio (audio_timesteps). For 'valid' the audio is
+      extended to include the tail of the impulse response (audio_timesteps +
+      ir_timesteps - 1).
+    delay_compensation: Samples to crop from start of output audio to compensate
+      for group delay of the impulse response. If delay_compensation < 0 it
+      defaults to automatically calculating a constant group delay of the
+      windowed linear phase filter from frequency_impulse_response().
+  Returns:
+    Tensor of cropped and shifted audio.
+  Raises:
+    ValueError: If padding is not either 'valid' or 'same'.
+  """
+  # Crop the output.
+  if padding == 'valid':
+    crop_size = ir_size + audio_size - 1
+  elif padding == 'same':
+    crop_size = audio_size
+  else:
+    raise ValueError('Padding must be \'valid\' or \'same\', instead '
+                     'of {}.'.format(padding))
+
+  # Compensate for the group delay of the filter by trimming the front.
+  # For an impulse response produced by frequency_impulse_response(),
+  # the group delay is constant because the filter is linear phase.
+  total_size = int(audio.shape[-1])
+  crop = total_size - crop_size
+  start = (ir_size // 2 if delay_compensation < 0 else delay_compensation)
+  end = crop - start
+  return audio[:, start:-end]
+
+
+def fft_convolve(audio,
+                 impulse_response): # B, n_frames, 2*(n_mags-1)
+    """Filter audio with frames of time-varying impulse responses.
+    Time-varying filter. Given audio [batch, n_samples], and a series of impulse
+    responses [batch, n_frames, n_impulse_response], splits the audio into frames,
+    applies filters, and then overlap-and-adds audio back together.
+    Applies non-windowed non-overlapping STFT/ISTFT to efficiently compute
+    convolution for large impulse response sizes.
+    Args:
+        audio: Input audio. Tensor of shape [batch, audio_timesteps].
+        impulse_response: Finite impulse response to convolve. Can either be a 2-D
+        Tensor of shape [batch, ir_size], or a 3-D Tensor of shape [batch,
+        ir_frames, ir_size]. A 2-D tensor will apply a single linear
+        time-invariant filter to the audio. A 3-D Tensor will apply a linear
+        time-varying filter. Automatically chops the audio into equally shaped
+        blocks to match ir_frames.
+    Returns:
+        audio_out: Convolved audio. Tensor of shape
+            [batch, audio_timesteps].
+    """
+    # Add a frame dimension to impulse response if it doesn't have one.
+    ir_shape = impulse_response.size() 
+    if len(ir_shape) == 2:
+        impulse_response = impulse_response.unsqueeze(1)
+        ir_shape = impulse_response.size()
+
+    # Get shapes of audio and impulse response.
+    batch_size_ir, n_ir_frames, ir_size = ir_shape
+    batch_size, audio_size = audio.size() # B, T
+
+    # Validate that batch sizes match.
+    if batch_size != batch_size_ir:
+        raise ValueError('Batch size of audio ({}) and impulse response ({}) must '
+                        'be the same.'.format(batch_size, batch_size_ir))
+
+    # Cut audio into 50% overlapped frames (center padding).
+    hop_size = int(audio_size / n_ir_frames)
+    frame_size = 2 * hop_size    
+    audio_frames = F.pad(audio, (hop_size, hop_size)).unfold(1, frame_size, hop_size)
+    
+    # Apply Bartlett (triangular) window
+    window = torch.bartlett_window(frame_size).to(audio_frames)
+    audio_frames = audio_frames * window
+    
+    # Pad and FFT the audio and impulse responses.
+    fft_size = get_fft_size(frame_size, ir_size, power_of_2=False)
+    audio_fft = torch.fft.rfft(audio_frames, fft_size)
+    ir_fft = torch.fft.rfft(torch.cat((impulse_response,impulse_response[:,-1:,:]),1), fft_size)
+    
+    # Multiply the FFTs (same as convolution in time).
+    audio_ir_fft = torch.multiply(audio_fft, ir_fft)
+
+    # Take the IFFT to resynthesize audio.
+    audio_frames_out = torch.fft.irfft(audio_ir_fft, fft_size)
+    
+    # Overlap Add
+    batch_size, n_audio_frames, frame_size = audio_frames_out.size() # # B, n_frames+1, 2*(hop_size+n_mags-1)-1
+    fold = torch.nn.Fold(output_size=(1, (n_audio_frames - 1) * hop_size + frame_size),kernel_size=(1, frame_size),stride=(1, hop_size))
+    output_signal = fold(audio_frames_out.transpose(1, 2)).squeeze(1).squeeze(1)
+    
+    # Crop and shift the output audio.
+    output_signal = crop_and_compensate_delay(output_signal[:,hop_size:], audio_size, ir_size)
+    return output_signal
+    
+
+def apply_window_to_impulse_response(impulse_response, # B, n_frames, 2*(n_mag-1)
+                                     window_size: int = 0,
+                                     causal: bool = False):
+    """Apply a window to an impulse response and put in causal form.
+    Args:
+        impulse_response: A series of impulse responses frames to window, of shape
+        [batch, n_frames, ir_size]. ---------> ir_size means size of filter_bank ??????
+        
+        window_size: Size of the window to apply in the time domain. If window_size
+        is less than 1, it defaults to the impulse_response size.
+        causal: Impulse response input is in causal form (peak in the middle).
+    Returns:
+        impulse_response: Windowed impulse response in causal form, with last
+        dimension cropped to window_size if window_size is greater than 0 and less
+        than ir_size.
+    """
+    
+    # If IR is in causal form, put it in zero-phase form.
+    if causal:
+        impulse_response = torch.fftshift(impulse_response, axes=-1)
+    
+    # Get a window for better time/frequency resolution than rectangular.
+    # Window defaults to IR size, cannot be bigger.
+    ir_size = int(impulse_response.size(-1))
+    if (window_size <= 0) or (window_size > ir_size):
+        window_size = ir_size
+    window = nn.Parameter(torch.hann_window(window_size), requires_grad = False).to(impulse_response)
+    
+    # Zero pad the window and put in in zero-phase form.
+    padding = ir_size - window_size
+    if padding > 0:
+        half_idx = (window_size + 1) // 2
+        window = torch.cat([window[half_idx:],
+                            torch.zeros([padding]),
+                            window[:half_idx]], axis=0)
+    else:
+        window = window.roll(window.size(-1)//2, -1)
+        
+    # Apply the window, to get new IR (both in zero-phase form).
+    window = window.unsqueeze(0)
+    impulse_response = impulse_response * window
+    
+    # Put IR in causal form and trim zero padding.
+    if padding > 0:
+        first_half_start = (ir_size - (half_idx - 1)) + 1
+        second_half_end = half_idx + 1
+        impulse_response = torch.cat([impulse_response[..., first_half_start:],
+                                    impulse_response[..., :second_half_end]],
+                                    dim=-1)
+    else:
+        impulse_response = impulse_response.roll(impulse_response.size(-1)//2, -1)
+
+    return impulse_response
+
+
+def apply_dynamic_window_to_impulse_response(impulse_response,  # B, n_frames, 2*(n_mag-1) or 2*n_mag-1
+                                             half_width_frames):        # B，n_frames, 1
+    ir_size = int(impulse_response.size(-1)) # 2*(n_mag -1) or 2*n_mag-1
+    
+    window = torch.arange(-(ir_size // 2), (ir_size + 1) // 2).to(impulse_response) / half_width_frames 
+    window[window > 1] = 0
+    window = (1 + torch.cos(np.pi * window)) / 2 # B, n_frames, 2*(n_mag -1) or 2*n_mag-1
+    
+    impulse_response = impulse_response.roll(ir_size // 2, -1)
+    impulse_response = impulse_response * window
+    
+    return impulse_response
+    
+        
+def frequency_impulse_response(magnitudes,
+                               hann_window = True,
+                               half_width_frames = None):
+                               
+    # Get the IR
+    impulse_response = torch.fft.irfft(magnitudes) # B, n_frames, 2*(n_mags-1)
+    
+    # Window and put in causal form.
+    if hann_window:
+        if half_width_frames is None:
+            impulse_response = apply_window_to_impulse_response(impulse_response)
+        else:
+            impulse_response = apply_dynamic_window_to_impulse_response(impulse_response, half_width_frames)
+    else:
+        impulse_response = impulse_response.roll(impulse_response.size(-1) // 2, -1)
+       
+    return impulse_response
+
+
+def frequency_filter(audio,
+                     magnitudes,
+                     hann_window=True,
+                     half_width_frames=None):
+
+    impulse_response = frequency_impulse_response(magnitudes, hann_window, half_width_frames)
+    
+    return fft_convolve(audio, impulse_response)
+    

DDSP-SVC/ddsp/loss.py

@@ -0,0 +1,57 @@
+import numpy as np
+
+import torch
+import torch.nn as nn
+import torchaudio
+from torch.nn import functional as F
+from .core import upsample
+        
+class SSSLoss(nn.Module):
+    """
+    Single-scale Spectral Loss. 
+    """
+
+    def __init__(self, n_fft=111, alpha=1.0, overlap=0, eps=1e-7):
+        super().__init__()
+        self.n_fft = n_fft
+        self.alpha = alpha
+        self.eps = eps
+        self.hop_length = int(n_fft * (1 - overlap))  # 25% of the length
+        self.spec = torchaudio.transforms.Spectrogram(n_fft=self.n_fft, hop_length=self.hop_length, power=1, normalized=True, center=False)
+        
+    def forward(self, x_true, x_pred):
+        S_true = self.spec(x_true) + self.eps
+        S_pred = self.spec(x_pred) + self.eps
+        
+        converge_term = torch.mean(torch.linalg.norm(S_true - S_pred, dim = (1, 2)) / torch.linalg.norm(S_true + S_pred, dim = (1, 2)))
+        
+        log_term = F.l1_loss(S_true.log(), S_pred.log())
+
+        loss = converge_term + self.alpha * log_term
+        return loss
+        
+        
+class RSSLoss(nn.Module):
+    '''
+    Random-scale Spectral Loss.
+    '''
+    
+    def __init__(self, fft_min, fft_max, n_scale, alpha=1.0, overlap=0, eps=1e-7, device='cuda'):
+        super().__init__()
+        self.fft_min = fft_min
+        self.fft_max = fft_max
+        self.n_scale = n_scale
+        self.lossdict = {}
+        for n_fft in range(fft_min, fft_max):
+            self.lossdict[n_fft] = SSSLoss(n_fft, alpha, overlap, eps).to(device)
+        
+    def forward(self, x_pred, x_true):
+        value = 0.
+        n_ffts = torch.randint(self.fft_min, self.fft_max, (self.n_scale,))
+        for n_fft in n_ffts:
+            loss_func = self.lossdict[int(n_fft)]
+            value += loss_func(x_true, x_pred)
+        return value / self.n_scale
+            
+        
+    

DDSP-SVC/ddsp/pcmer.py

@@ -0,0 +1,380 @@
+import torch
+
+from torch import nn
+import math
+from functools import partial
+from einops import rearrange, repeat
+
+from local_attention import LocalAttention
+import torch.nn.functional as F
+#import fast_transformers.causal_product.causal_product_cuda
+
+def softmax_kernel(data, *, projection_matrix, is_query, normalize_data=True, eps=1e-4, device = None):
+    b, h, *_ = data.shape
+    # (batch size, head, length, model_dim)
+
+    # normalize model dim
+    data_normalizer = (data.shape[-1] ** -0.25) if normalize_data else 1.
+
+    # what is ration?, projection_matrix.shape[0] --> 266
+    
+    ratio = (projection_matrix.shape[0] ** -0.5)
+
+    projection = repeat(projection_matrix, 'j d -> b h j d', b = b, h = h)
+    projection = projection.type_as(data)
+
+    #data_dash = w^T x
+    data_dash = torch.einsum('...id,...jd->...ij', (data_normalizer * data), projection)
+
+    
+    # diag_data = D**2 
+    diag_data = data ** 2
+    diag_data = torch.sum(diag_data, dim=-1)
+    diag_data = (diag_data / 2.0) * (data_normalizer ** 2)
+    diag_data = diag_data.unsqueeze(dim=-1)
+    
+    #print ()
+    if is_query:
+        data_dash = ratio * (
+            torch.exp(data_dash - diag_data -
+                    torch.max(data_dash, dim=-1, keepdim=True).values) + eps)
+    else:
+        data_dash = ratio * (
+            torch.exp(data_dash - diag_data + eps))#- torch.max(data_dash)) + eps)
+
+    return data_dash.type_as(data)
+
+def orthogonal_matrix_chunk(cols, qr_uniform_q = False, device = None):
+    unstructured_block = torch.randn((cols, cols), device = device)
+    q, r = torch.linalg.qr(unstructured_block.cpu(), mode='reduced')
+    q, r = map(lambda t: t.to(device), (q, r))
+
+    # proposed by @Parskatt
+    # to make sure Q is uniform https://arxiv.org/pdf/math-ph/0609050.pdf
+    if qr_uniform_q:
+        d = torch.diag(r, 0)
+        q *= d.sign()
+    return q.t()
+def exists(val):
+    return val is not None
+
+def empty(tensor):
+    return tensor.numel() == 0
+
+def default(val, d):
+    return val if exists(val) else d
+
+def cast_tuple(val):
+    return (val,) if not isinstance(val, tuple) else val
+
+class PCmer(nn.Module):
+    """The encoder that is used in the Transformer model."""
+    
+    def __init__(self, 
+                num_layers,
+                num_heads,
+                dim_model,
+                dim_keys,
+                dim_values,
+                residual_dropout,
+                attention_dropout):
+        super().__init__()
+        self.num_layers = num_layers
+        self.num_heads = num_heads
+        self.dim_model = dim_model
+        self.dim_values = dim_values
+        self.dim_keys = dim_keys
+        self.residual_dropout = residual_dropout
+        self.attention_dropout = attention_dropout
+
+        self._layers = nn.ModuleList([_EncoderLayer(self) for _ in range(num_layers)])
+        
+    #  METHODS  ########################################################################################################
+    
+    def forward(self, phone, mask=None):
+        
+        # apply all layers to the input
+        for (i, layer) in enumerate(self._layers):
+            phone = layer(phone, mask)
+        # provide the final sequence
+        return phone
+
+
+# ==================================================================================================================== #
+#  CLASS  _ E N C O D E R  L A Y E R                                                                                   #
+# ==================================================================================================================== #
+
+
+class _EncoderLayer(nn.Module):
+    """One layer of the encoder.
+    
+    Attributes:
+        attn: (:class:`mha.MultiHeadAttention`): The attention mechanism that is used to read the input sequence.
+        feed_forward (:class:`ffl.FeedForwardLayer`): The feed-forward layer on top of the attention mechanism.
+    """
+    
+    def __init__(self, parent: PCmer):
+        """Creates a new instance of ``_EncoderLayer``.
+        
+        Args:
+            parent (Encoder): The encoder that the layers is created for.
+        """
+        super().__init__()
+        
+        
+        self.conformer = ConformerConvModule(parent.dim_model)
+        self.norm = nn.LayerNorm(parent.dim_model)
+        self.dropout = nn.Dropout(parent.residual_dropout)
+        
+        # selfatt -> fastatt: performer!
+        self.attn = SelfAttention(dim = parent.dim_model,
+                                  heads = parent.num_heads,
+                                  causal = False)
+        
+    #  METHODS  ########################################################################################################
+
+    def forward(self, phone, mask=None):
+        
+        # compute attention sub-layer
+        phone = phone + (self.attn(self.norm(phone), mask=mask))
+        
+        phone = phone + (self.conformer(phone))
+        
+        return phone 
+
+def calc_same_padding(kernel_size):
+    pad = kernel_size // 2
+    return (pad, pad - (kernel_size + 1) % 2)
+
+# helper classes
+
+class Swish(nn.Module):
+    def forward(self, x):
+        return x * x.sigmoid()
+
+class Transpose(nn.Module):
+    def __init__(self, dims):
+        super().__init__()
+        assert len(dims) == 2, 'dims must be a tuple of two dimensions'
+        self.dims = dims
+
+    def forward(self, x):
+        return x.transpose(*self.dims)
+
+class GLU(nn.Module):
+    def __init__(self, dim):
+        super().__init__()
+        self.dim = dim
+
+    def forward(self, x):
+        out, gate = x.chunk(2, dim=self.dim)
+        return out * gate.sigmoid()
+
+class DepthWiseConv1d(nn.Module):
+    def __init__(self, chan_in, chan_out, kernel_size, padding):
+        super().__init__()
+        self.padding = padding
+        self.conv = nn.Conv1d(chan_in, chan_out, kernel_size, groups = chan_in)
+
+    def forward(self, x):
+        x = F.pad(x, self.padding)
+        return self.conv(x)
+
+class ConformerConvModule(nn.Module):
+    def __init__(
+        self,
+        dim,
+        causal = False,
+        expansion_factor = 2,
+        kernel_size = 31,
+        dropout = 0.):
+        super().__init__()
+
+        inner_dim = dim * expansion_factor
+        padding = calc_same_padding(kernel_size) if not causal else (kernel_size - 1, 0)
+
+        self.net = nn.Sequential(
+            nn.LayerNorm(dim),
+            Transpose((1, 2)),
+            nn.Conv1d(dim, inner_dim * 2, 1),
+            GLU(dim=1),
+            DepthWiseConv1d(inner_dim, inner_dim, kernel_size = kernel_size, padding = padding),
+            #nn.BatchNorm1d(inner_dim) if not causal else nn.Identity(),
+            Swish(),
+            nn.Conv1d(inner_dim, dim, 1),
+            Transpose((1, 2)),
+            nn.Dropout(dropout)
+        )
+
+    def forward(self, x):
+        return self.net(x)
+
+def linear_attention(q, k, v):
+    if v is None:
+        #print (k.size(), q.size())
+        out = torch.einsum('...ed,...nd->...ne', k, q)
+        return out
+
+    else:
+        k_cumsum = k.sum(dim = -2) 
+        #k_cumsum = k.sum(dim = -2)
+        D_inv = 1. / (torch.einsum('...nd,...d->...n', q, k_cumsum.type_as(q)) + 1e-8)
+
+        context = torch.einsum('...nd,...ne->...de', k, v)
+        #print ("TRUEEE: ", context.size(), q.size(), D_inv.size())
+        out = torch.einsum('...de,...nd,...n->...ne', context, q, D_inv)
+        return out
+
+def gaussian_orthogonal_random_matrix(nb_rows, nb_columns, scaling = 0, qr_uniform_q = False, device = None):
+    nb_full_blocks = int(nb_rows / nb_columns)
+    #print (nb_full_blocks)
+    block_list = []
+
+    for _ in range(nb_full_blocks):
+        q = orthogonal_matrix_chunk(nb_columns, qr_uniform_q = qr_uniform_q, device = device)
+        block_list.append(q)
+    # block_list[n] is a orthogonal matrix ... (model_dim * model_dim)
+    #print (block_list[0].size(), torch.einsum('...nd,...nd->...n', block_list[0], torch.roll(block_list[0],1,1)))
+    #print (nb_rows, nb_full_blocks, nb_columns)
+    remaining_rows = nb_rows - nb_full_blocks * nb_columns
+    #print (remaining_rows)
+    if remaining_rows > 0:
+        q = orthogonal_matrix_chunk(nb_columns, qr_uniform_q = qr_uniform_q, device = device)
+        #print (q[:remaining_rows].size())
+        block_list.append(q[:remaining_rows])
+
+    final_matrix = torch.cat(block_list)
+    
+    if scaling == 0:
+        multiplier = torch.randn((nb_rows, nb_columns), device = device).norm(dim = 1)
+    elif scaling == 1:
+        multiplier = math.sqrt((float(nb_columns))) * torch.ones((nb_rows,), device = device)
+    else:
+        raise ValueError(f'Invalid scaling {scaling}')
+
+    return torch.diag(multiplier) @ final_matrix
+
+class FastAttention(nn.Module):
+    def __init__(self, dim_heads, nb_features = None, ortho_scaling = 0, causal = False, generalized_attention = False, kernel_fn = nn.ReLU(), qr_uniform_q = False, no_projection = False):
+        super().__init__()
+        nb_features = default(nb_features, int(dim_heads * math.log(dim_heads)))
+
+        self.dim_heads = dim_heads
+        self.nb_features = nb_features
+        self.ortho_scaling = ortho_scaling
+
+        self.create_projection = partial(gaussian_orthogonal_random_matrix, nb_rows = self.nb_features, nb_columns = dim_heads, scaling = ortho_scaling, qr_uniform_q = qr_uniform_q)
+        projection_matrix = self.create_projection()
+        self.register_buffer('projection_matrix', projection_matrix)
+
+        self.generalized_attention = generalized_attention
+        self.kernel_fn = kernel_fn
+
+        # if this is turned on, no projection will be used
+        # queries and keys will be softmax-ed as in the original efficient attention paper
+        self.no_projection = no_projection
+
+        self.causal = causal
+        if causal:
+            try:
+                import fast_transformers.causal_product.causal_product_cuda
+                self.causal_linear_fn = partial(causal_linear_attention)
+            except ImportError:
+                print('unable to import cuda code for auto-regressive Performer. will default to the memory inefficient non-cuda version')
+                self.causal_linear_fn = causal_linear_attention_noncuda
+    @torch.no_grad()
+    def redraw_projection_matrix(self):
+        projections = self.create_projection()
+        self.projection_matrix.copy_(projections)
+        del projections
+
+    def forward(self, q, k, v):
+        device = q.device
+
+        if self.no_projection:
+            q = q.softmax(dim = -1)
+            k = torch.exp(k) if self.causal else k.softmax(dim = -2)
+
+        elif self.generalized_attention:
+            create_kernel = partial(generalized_kernel, kernel_fn = self.kernel_fn, projection_matrix = self.projection_matrix, device = device)
+            q, k = map(create_kernel, (q, k))
+
+        else:
+            create_kernel = partial(softmax_kernel, projection_matrix = self.projection_matrix, device = device)
+            
+            q = create_kernel(q, is_query = True)
+            k = create_kernel(k, is_query = False)
+
+        attn_fn = linear_attention if not self.causal else self.causal_linear_fn
+        if v is None:
+            out = attn_fn(q, k, None)
+            return out
+        else:
+            out = attn_fn(q, k, v)
+            return out
+class SelfAttention(nn.Module):
+    def __init__(self, dim, causal = False, heads = 8, dim_head = 64, local_heads = 0, local_window_size = 256, nb_features = None, feature_redraw_interval = 1000, generalized_attention = False, kernel_fn = nn.ReLU(), qr_uniform_q = False, dropout = 0., no_projection = False):
+        super().__init__()
+        assert dim % heads == 0, 'dimension must be divisible by number of heads'
+        dim_head = default(dim_head, dim // heads)
+        inner_dim = dim_head * heads
+        self.fast_attention = FastAttention(dim_head, nb_features, causal = causal, generalized_attention = generalized_attention, kernel_fn = kernel_fn, qr_uniform_q = qr_uniform_q, no_projection = no_projection)
+
+        self.heads = heads
+        self.global_heads = heads - local_heads
+        self.local_attn = LocalAttention(window_size = local_window_size, causal = causal, autopad = True, dropout = dropout, look_forward = int(not causal), rel_pos_emb_config = (dim_head, local_heads)) if local_heads > 0 else None
+
+        #print (heads, nb_features, dim_head)
+        #name_embedding = torch.zeros(110, heads, dim_head, dim_head)
+        #self.name_embedding = nn.Parameter(name_embedding, requires_grad=True)
+        
+
+        self.to_q = nn.Linear(dim, inner_dim)
+        self.to_k = nn.Linear(dim, inner_dim)
+        self.to_v = nn.Linear(dim, inner_dim)
+        self.to_out = nn.Linear(inner_dim, dim)
+        self.dropout = nn.Dropout(dropout)
+
+    @torch.no_grad()
+    def redraw_projection_matrix(self):
+        self.fast_attention.redraw_projection_matrix()
+        #torch.nn.init.zeros_(self.name_embedding)
+        #print (torch.sum(self.name_embedding))
+    def forward(self, x, context = None, mask = None, context_mask = None, name=None, inference=False, **kwargs):
+        b, n, _, h, gh = *x.shape, self.heads, self.global_heads
+        
+        cross_attend = exists(context)
+
+        context = default(context, x)
+        context_mask = default(context_mask, mask) if not cross_attend else context_mask
+        #print (torch.sum(self.name_embedding))
+        q, k, v = self.to_q(x), self.to_k(context), self.to_v(context)
+
+        q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = h), (q, k, v))
+        (q, lq), (k, lk), (v, lv) = map(lambda t: (t[:, :gh], t[:, gh:]), (q, k, v))
+
+        attn_outs = []
+        #print (name)
+        #print (self.name_embedding[name].size())
+        if not empty(q):
+            if exists(context_mask):
+                global_mask = context_mask[:, None, :, None]
+                v.masked_fill_(~global_mask, 0.)
+            if cross_attend:
+                pass
+                #print (torch.sum(self.name_embedding))
+                #out = self.fast_attention(q,self.name_embedding[name],None)
+                #print (torch.sum(self.name_embedding[...,-1:]))
+            else:
+                out = self.fast_attention(q, k, v)
+            attn_outs.append(out)
+
+        if not empty(lq):
+            assert not cross_attend, 'local attention is not compatible with cross attention'
+            out = self.local_attn(lq, lk, lv, input_mask = mask)
+            attn_outs.append(out)
+
+        out = torch.cat(attn_outs, dim = 1)
+        out = rearrange(out, 'b h n d -> b n (h d)')
+        out =  self.to_out(out)
+        return self.dropout(out)

DDSP-SVC/ddsp/unit2control.py

@@ -0,0 +1,86 @@
+import gin
+
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.nn.utils import weight_norm
+
+from .pcmer import PCmer
+
+
+def split_to_dict(tensor, tensor_splits):
+    """Split a tensor into a dictionary of multiple tensors."""
+    labels = []
+    sizes = []
+
+    for k, v in tensor_splits.items():
+        labels.append(k)
+        sizes.append(v)
+
+    tensors = torch.split(tensor, sizes, dim=-1)
+    return dict(zip(labels, tensors))
+
+
+class Unit2Control(nn.Module):
+    def __init__(
+            self,
+            input_channel,
+            n_spk,
+            output_splits):
+        super().__init__()
+        self.output_splits = output_splits
+        self.f0_embed = nn.Linear(1, 256)
+        self.phase_embed = nn.Linear(1, 256)
+        self.volume_embed = nn.Linear(1, 256)
+        self.n_spk = n_spk
+        if n_spk is not None and n_spk > 1:
+            self.spk_embed = nn.Embedding(n_spk, 256)
+            
+        # conv in stack
+        self.stack = nn.Sequential(
+                nn.Conv1d(input_channel, 256, 3, 1, 1),
+                nn.GroupNorm(4, 256),
+                nn.LeakyReLU(),
+                nn.Conv1d(256, 256, 3, 1, 1)) 
+
+        # transformer
+        self.decoder = PCmer(
+            num_layers=3,
+            num_heads=8,
+            dim_model=256,
+            dim_keys=256,
+            dim_values=256,
+            residual_dropout=0.1,
+            attention_dropout=0.1)
+        self.norm = nn.LayerNorm(256)
+
+        # out
+        self.n_out = sum([v for k, v in output_splits.items()])
+        self.dense_out = weight_norm(
+            nn.Linear(256, self.n_out))
+
+    def forward(self, units, f0, phase, volume, spk_id = None, spk_mix_dict = None):
+        
+        '''
+        input: 
+            B x n_frames x n_unit
+        return: 
+            dict of B x n_frames x feat
+        '''
+
+        x = self.stack(units.transpose(1,2)).transpose(1,2)
+        x = x + self.f0_embed((1+ f0 / 700).log()) + self.phase_embed(phase / np.pi) + self.volume_embed(volume)
+        if self.n_spk is not None and self.n_spk > 1:
+            if spk_mix_dict is not None:
+                for k, v in spk_mix_dict.items():
+                    spk_id_torch = torch.LongTensor(np.array([[k]])).to(units.device)
+                    x = x + v * self.spk_embed(spk_id_torch - 1)
+            else:
+                x = x + self.spk_embed(spk_id - 1)
+        x = self.decoder(x)
+        x = self.norm(x)
+        e = self.dense_out(x)
+        controls = split_to_dict(e, self.output_splits)
+    
+        return controls 
+

DDSP-SVC/ddsp/vocoder.py

@@ -0,0 +1,652 @@
+import os
+import numpy as np
+import yaml
+import torch
+import torch.nn.functional as F
+import pyworld as pw
+import parselmouth
+import torchcrepe
+import resampy
+from transformers import HubertModel, Wav2Vec2FeatureExtractor
+from fairseq import checkpoint_utils
+from encoder.hubert.model import HubertSoft
+from torch.nn.modules.utils import consume_prefix_in_state_dict_if_present
+from torchaudio.transforms import Resample
+from .unit2control import Unit2Control
+from .core import frequency_filter, upsample, remove_above_fmax, MaskedAvgPool1d, MedianPool1d
+import time
+
+CREPE_RESAMPLE_KERNEL = {}
+
+class F0_Extractor:
+    def __init__(self, f0_extractor, sample_rate = 44100, hop_size = 512, f0_min = 65, f0_max = 800):
+        self.f0_extractor = f0_extractor
+        self.sample_rate = sample_rate
+        self.hop_size = hop_size
+        self.f0_min = f0_min
+        self.f0_max = f0_max
+        if f0_extractor == 'crepe':
+            key_str = str(sample_rate)
+            if key_str not in CREPE_RESAMPLE_KERNEL:
+                CREPE_RESAMPLE_KERNEL[key_str] = Resample(sample_rate, 16000, lowpass_filter_width = 128)
+            self.resample_kernel = CREPE_RESAMPLE_KERNEL[key_str]
+                
+    def extract(self, audio, uv_interp = False, device = None, silence_front = 0): # audio: 1d numpy array
+        # extractor start time
+        n_frames = int(len(audio) // self.hop_size) + 1
+                
+        start_frame = int(silence_front * self.sample_rate / self.hop_size)
+        real_silence_front = start_frame * self.hop_size / self.sample_rate
+        audio = audio[int(np.round(real_silence_front * self.sample_rate)) : ]
+        
+        # extract f0 using parselmouth
+        if self.f0_extractor == 'parselmouth':
+            f0 = parselmouth.Sound(audio, self.sample_rate).to_pitch_ac(
+                time_step = self.hop_size / self.sample_rate, 
+                voicing_threshold = 0.6,
+                pitch_floor = self.f0_min, 
+                pitch_ceiling = self.f0_max).selected_array['frequency']
+            pad_size = start_frame + (int(len(audio) // self.hop_size) - len(f0) + 1) // 2
+            f0 = np.pad(f0,(pad_size, n_frames - len(f0) - pad_size))
+            
+        # extract f0 using dio
+        elif self.f0_extractor == 'dio':
+            _f0, t = pw.dio(
+                audio.astype('double'), 
+                self.sample_rate, 
+                f0_floor = self.f0_min, 
+                f0_ceil = self.f0_max, 
+                channels_in_octave=2, 
+                frame_period = (1000 * self.hop_size / self.sample_rate))
+            f0 = pw.stonemask(audio.astype('double'), _f0, t, self.sample_rate)
+            f0 = np.pad(f0.astype('float'), (start_frame, n_frames - len(f0) - start_frame))
+        
+        # extract f0 using harvest
+        elif self.f0_extractor == 'harvest':
+            f0, _ = pw.harvest(
+                audio.astype('double'), 
+                self.sample_rate, 
+                f0_floor = self.f0_min, 
+                f0_ceil = self.f0_max, 
+                frame_period = (1000 * self.hop_size / self.sample_rate))
+            f0 = np.pad(f0.astype('float'), (start_frame, n_frames - len(f0) - start_frame))
+        
+        # extract f0 using crepe        
+        elif self.f0_extractor == 'crepe':
+            if device is None:
+                device = 'cuda' if torch.cuda.is_available() else 'cpu'
+            resample_kernel = self.resample_kernel.to(device)
+            wav16k_torch = resample_kernel(torch.FloatTensor(audio).unsqueeze(0).to(device))
+            
+            f0, pd = torchcrepe.predict(wav16k_torch, 16000, 80, self.f0_min, self.f0_max, pad=True, model='full', batch_size=512, device=device, return_periodicity=True)
+            pd = MedianPool1d(pd, 4)
+            f0 = torchcrepe.threshold.At(0.05)(f0, pd)
+            f0 = MaskedAvgPool1d(f0, 4)
+            
+            f0 = f0.squeeze(0).cpu().numpy()
+            f0 = np.array([f0[int(min(int(np.round(n * self.hop_size / self.sample_rate / 0.005)), len(f0) - 1))] for n in range(n_frames - start_frame)])
+            f0 = np.pad(f0, (start_frame, 0))
+           
+        else:
+            raise ValueError(f" [x] Unknown f0 extractor: {f0_extractor}")
+                    
+        # interpolate the unvoiced f0 
+        if uv_interp:
+            uv = f0 == 0
+            if len(f0[~uv]) > 0:
+                f0[uv] = np.interp(np.where(uv)[0], np.where(~uv)[0], f0[~uv])
+            f0[f0 < self.f0_min] = self.f0_min
+        return f0
+
+
+class Volume_Extractor:
+    def __init__(self, hop_size = 512):
+        self.hop_size = hop_size
+        
+    def extract(self, audio): # audio: 1d numpy array
+        n_frames = int(len(audio) // self.hop_size) + 1
+        audio2 = audio ** 2
+        audio2 = np.pad(audio2, (int(self.hop_size // 2), int((self.hop_size + 1) // 2)), mode = 'reflect')
+        volume = np.array([np.mean(audio2[int(n * self.hop_size) : int((n + 1) * self.hop_size)]) for n in range(n_frames)])
+        volume = np.sqrt(volume)
+        return volume
+    
+         
+class Units_Encoder:
+    def __init__(self, encoder, encoder_ckpt, encoder_sample_rate = 16000, encoder_hop_size = 320, device = None,
+                 cnhubertsoft_gate=10):
+        if device is None:
+            device = 'cuda' if torch.cuda.is_available() else 'cpu'
+        self.device = device
+        
+        is_loaded_encoder = False
+        if encoder == 'hubertsoft':
+            self.model = Audio2HubertSoft(encoder_ckpt).to(device)
+            is_loaded_encoder = True
+        if encoder == 'hubertbase':
+            self.model = Audio2HubertBase(encoder_ckpt, device=device)
+            is_loaded_encoder = True
+        if encoder == 'hubertbase768':
+            self.model = Audio2HubertBase768(encoder_ckpt, device=device)
+            is_loaded_encoder = True
+        if encoder == 'contentvec':
+            self.model = Audio2ContentVec(encoder_ckpt, device=device)
+            is_loaded_encoder = True
+        if encoder == 'contentvec768':
+            self.model = Audio2ContentVec768(encoder_ckpt, device=device)
+            is_loaded_encoder = True
+        if encoder == 'contentvec768l12':
+            self.model = Audio2ContentVec768L12(encoder_ckpt, device=device)
+            is_loaded_encoder = True
+        if encoder == 'cnhubertsoftfish':
+            self.model = CNHubertSoftFish(encoder_ckpt, device=device, gate_size=cnhubertsoft_gate)
+            is_loaded_encoder = True
+        if not is_loaded_encoder:
+            raise ValueError(f" [x] Unknown units encoder: {encoder}")
+            
+        self.resample_kernel = {}
+        self.encoder_sample_rate = encoder_sample_rate
+        self.encoder_hop_size = encoder_hop_size
+        
+    def encode(self, 
+                audio, # B, T
+                sample_rate,
+                hop_size): 
+        
+        # resample
+        if sample_rate == self.encoder_sample_rate:
+            audio_res = audio
+        else:
+            key_str = str(sample_rate)
+            if key_str not in self.resample_kernel:
+                self.resample_kernel[key_str] = Resample(sample_rate, self.encoder_sample_rate, lowpass_filter_width = 128).to(self.device)
+            audio_res = self.resample_kernel[key_str](audio)
+        
+        # encode
+        if audio_res.size(-1) < self.encoder_hop_size:
+            audio_res = torch.nn.functional.pad(audio, (0, self.encoder_hop_size - audio_res.size(-1)))
+        units = self.model(audio_res)
+        
+        # alignment
+        n_frames = audio.size(-1) // hop_size + 1
+        ratio = (hop_size / sample_rate) / (self.encoder_hop_size / self.encoder_sample_rate)
+        index = torch.clamp(torch.round(ratio * torch.arange(n_frames).to(self.device)).long(), max = units.size(1) - 1)
+        units_aligned = torch.gather(units, 1, index.unsqueeze(0).unsqueeze(-1).repeat([1, 1, units.size(-1)]))
+        return units_aligned
+        
+class Audio2HubertSoft(torch.nn.Module):
+    def __init__(self, path, h_sample_rate = 16000, h_hop_size = 320):
+        super().__init__()
+        print(' [Encoder Model] HuBERT Soft')
+        self.hubert = HubertSoft()
+        print(' [Loading] ' + path)
+        checkpoint = torch.load(path)
+        consume_prefix_in_state_dict_if_present(checkpoint, "module.")
+        self.hubert.load_state_dict(checkpoint)
+        self.hubert.eval()
+     
+    def forward(self, 
+                audio): # B, T
+        with torch.inference_mode():  
+            units = self.hubert.units(audio.unsqueeze(1))
+            return units
+
+
+class Audio2ContentVec():
+    def __init__(self, path, h_sample_rate=16000, h_hop_size=320, device='cpu'):
+        self.device = device
+        print(' [Encoder Model] Content Vec')
+        print(' [Loading] ' + path)
+        self.models, self.saved_cfg, self.task = checkpoint_utils.load_model_ensemble_and_task([path], suffix="", )
+        self.hubert = self.models[0]
+        self.hubert = self.hubert.to(self.device)
+        self.hubert.eval()
+
+    def __call__(self,
+                 audio):  # B, T
+        # wav_tensor = torch.from_numpy(audio).to(self.device)
+        wav_tensor = audio
+        feats = wav_tensor.view(1, -1)
+        padding_mask = torch.BoolTensor(feats.shape).fill_(False)
+        inputs = {
+            "source": feats.to(wav_tensor.device),
+            "padding_mask": padding_mask.to(wav_tensor.device),
+            "output_layer": 9,  # layer 9
+        }
+        with torch.no_grad():
+            logits = self.hubert.extract_features(**inputs)
+            feats = self.hubert.final_proj(logits[0])
+        units = feats  # .transpose(2, 1)
+        return units
+
+
+class Audio2ContentVec768():
+    def __init__(self, path, h_sample_rate=16000, h_hop_size=320, device='cpu'):
+        self.device = device
+        print(' [Encoder Model] Content Vec')
+        print(' [Loading] ' + path)
+        self.models, self.saved_cfg, self.task = checkpoint_utils.load_model_ensemble_and_task([path], suffix="", )
+        self.hubert = self.models[0]
+        self.hubert = self.hubert.to(self.device)
+        self.hubert.eval()
+
+    def __call__(self,
+                 audio):  # B, T
+        # wav_tensor = torch.from_numpy(audio).to(self.device)
+        wav_tensor = audio
+        feats = wav_tensor.view(1, -1)
+        padding_mask = torch.BoolTensor(feats.shape).fill_(False)
+        inputs = {
+            "source": feats.to(wav_tensor.device),
+            "padding_mask": padding_mask.to(wav_tensor.device),
+            "output_layer": 9,  # layer 9
+        }
+        with torch.no_grad():
+            logits = self.hubert.extract_features(**inputs)
+            feats = logits[0]
+        units = feats  # .transpose(2, 1)
+        return units
+
+
+class Audio2ContentVec768L12():
+    def __init__(self, path, h_sample_rate=16000, h_hop_size=320, device='cpu'):
+        self.device = device
+        print(' [Encoder Model] Content Vec')
+        print(' [Loading] ' + path)
+        self.models, self.saved_cfg, self.task = checkpoint_utils.load_model_ensemble_and_task([path], suffix="", )
+        self.hubert = self.models[0]
+        self.hubert = self.hubert.to(self.device)
+        self.hubert.eval()
+
+    def __call__(self,
+                 audio):  # B, T
+        # wav_tensor = torch.from_numpy(audio).to(self.device)
+        wav_tensor = audio
+        feats = wav_tensor.view(1, -1)
+        padding_mask = torch.BoolTensor(feats.shape).fill_(False)
+        inputs = {
+            "source": feats.to(wav_tensor.device),
+            "padding_mask": padding_mask.to(wav_tensor.device),
+            "output_layer": 12,  # layer 12
+        }
+        with torch.no_grad():
+            logits = self.hubert.extract_features(**inputs)
+            feats = logits[0]
+        units = feats  # .transpose(2, 1)
+        return units    
+
+
+class CNHubertSoftFish(torch.nn.Module):
+    def __init__(self, path, h_sample_rate=16000, h_hop_size=320, device='cpu', gate_size=10):
+        super().__init__()
+        self.device = device
+        self.gate_size = gate_size
+
+        self.feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(
+            "./pretrain/TencentGameMate/chinese-hubert-base")
+        self.model = HubertModel.from_pretrained("./pretrain/TencentGameMate/chinese-hubert-base")
+        self.proj = torch.nn.Sequential(torch.nn.Dropout(0.1), torch.nn.Linear(768, 256))
+        # self.label_embedding = nn.Embedding(128, 256)
+
+        state_dict = torch.load(path, map_location=device)
+        self.load_state_dict(state_dict)
+
+    @torch.no_grad()
+    def forward(self, audio):
+        input_values = self.feature_extractor(
+            audio, sampling_rate=16000, return_tensors="pt"
+        ).input_values
+        input_values = input_values.to(self.model.device)
+
+        return self._forward(input_values[0])
+
+    @torch.no_grad()
+    def _forward(self, input_values):
+        features = self.model(input_values)
+        features = self.proj(features.last_hidden_state)
+
+        # Top-k gating
+        topk, indices = torch.topk(features, self.gate_size, dim=2)
+        features = torch.zeros_like(features).scatter(2, indices, topk)
+        features = features / features.sum(2, keepdim=True)
+
+        return features.to(self.device)  # .transpose(1, 2)
+
+    
+class Audio2HubertBase():
+    def __init__(self, path, h_sample_rate=16000, h_hop_size=320, device='cpu'):
+        self.device = device
+        print(' [Encoder Model] HuBERT Base')
+        print(' [Loading] ' + path)
+        self.models, self.saved_cfg, self.task = checkpoint_utils.load_model_ensemble_and_task([path], suffix="", )
+        self.hubert = self.models[0]
+        self.hubert = self.hubert.to(self.device)
+        self.hubert = self.hubert.float()
+        self.hubert.eval()
+
+    def __call__(self,
+                 audio):  # B, T
+        with torch.no_grad():
+            padding_mask = torch.BoolTensor(audio.shape).fill_(False)
+            inputs = {
+                "source": audio.to(self.device),
+                "padding_mask": padding_mask.to(self.device),
+                "output_layer": 9,  # layer 9
+            }
+            logits = self.hubert.extract_features(**inputs)
+            units = self.hubert.final_proj(logits[0])
+            return units
+
+
+class Audio2HubertBase768():
+    def __init__(self, path, h_sample_rate=16000, h_hop_size=320, device='cpu'):
+        self.device = device
+        print(' [Encoder Model] HuBERT Base')
+        print(' [Loading] ' + path)
+        self.models, self.saved_cfg, self.task = checkpoint_utils.load_model_ensemble_and_task([path], suffix="", )
+        self.hubert = self.models[0]
+        self.hubert = self.hubert.to(self.device)
+        self.hubert = self.hubert.float()
+        self.hubert.eval()
+
+    def __call__(self,
+                 audio):  # B, T
+        with torch.no_grad():
+            padding_mask = torch.BoolTensor(audio.shape).fill_(False)
+            inputs = {
+                "source": audio.to(self.device),
+                "padding_mask": padding_mask.to(self.device),
+                "output_layer": 9,  # layer 9
+            }
+            logits = self.hubert.extract_features(**inputs)
+            units = logits[0]
+            return units
+
+
+class DotDict(dict):
+    def __getattr__(*args):         
+        val = dict.get(*args)         
+        return DotDict(val) if type(val) is dict else val   
+
+    __setattr__ = dict.__setitem__    
+    __delattr__ = dict.__delitem__
+    
+def load_model(
+        model_path,
+        device='cpu'):
+    config_file = os.path.join(os.path.split(model_path)[0], 'config.yaml')
+    with open(config_file, "r") as config:
+        args = yaml.safe_load(config)
+    args = DotDict(args)
+    
+    # load model
+    model = None
+
+    if args.model.type == 'Sins':
+        model = Sins(
+            sampling_rate=args.data.sampling_rate,
+            block_size=args.data.block_size,
+            n_harmonics=args.model.n_harmonics,
+            n_mag_allpass=args.model.n_mag_allpass,
+            n_mag_noise=args.model.n_mag_noise,
+            n_unit=args.data.encoder_out_channels,
+            n_spk=args.model.n_spk)
+    
+    elif args.model.type == 'CombSub':
+        model = CombSub(
+            sampling_rate=args.data.sampling_rate,
+            block_size=args.data.block_size,
+            n_mag_allpass=args.model.n_mag_allpass,
+            n_mag_harmonic=args.model.n_mag_harmonic,
+            n_mag_noise=args.model.n_mag_noise,
+            n_unit=args.data.encoder_out_channels,
+            n_spk=args.model.n_spk)
+    
+    elif args.model.type == 'CombSubFast':
+        model = CombSubFast(
+            sampling_rate=args.data.sampling_rate,
+            block_size=args.data.block_size,
+            n_unit=args.data.encoder_out_channels,
+            n_spk=args.model.n_spk)
+            
+    else:
+        raise ValueError(f" [x] Unknown Model: {args.model.type}")
+    
+    print(' [Loading] ' + model_path)
+    ckpt = torch.load(model_path, map_location=torch.device(device))
+    model.to(device)
+    model.load_state_dict(ckpt['model'])
+    model.eval()
+    return model, args
+
+
+class Sins(torch.nn.Module):
+    def __init__(self, 
+            sampling_rate,
+            block_size,
+            n_harmonics,
+            n_mag_allpass,
+            n_mag_noise,
+            n_unit=256,
+            n_spk=1):
+        super().__init__()
+
+        print(' [DDSP Model] Sinusoids Additive Synthesiser')
+
+        # params
+        self.register_buffer("sampling_rate", torch.tensor(sampling_rate))
+        self.register_buffer("block_size", torch.tensor(block_size))
+        # Unit2Control
+        split_map = {
+            'amplitudes': n_harmonics,
+            'group_delay': n_mag_allpass,
+            'noise_magnitude': n_mag_noise,
+        }
+        self.unit2ctrl = Unit2Control(n_unit, n_spk, split_map)
+
+    def forward(self, units_frames, f0_frames, volume_frames, spk_id=None, spk_mix_dict=None, initial_phase=None, infer=True, max_upsample_dim=32):
+        '''
+            units_frames: B x n_frames x n_unit
+            f0_frames: B x n_frames x 1
+            volume_frames: B x n_frames x 1
+            spk_id: B x 1
+        '''
+        # exciter phase
+        f0 = upsample(f0_frames, self.block_size)
+        if infer:
+            x = torch.cumsum(f0.double() / self.sampling_rate, axis=1)
+        else:
+            x = torch.cumsum(f0 / self.sampling_rate, axis=1)
+        if initial_phase is not None:
+            x += initial_phase.to(x) / 2 / np.pi    
+        x = x - torch.round(x)
+        x = x.to(f0)
+        
+        phase = 2 * np.pi * x
+        phase_frames = phase[:, ::self.block_size, :]
+        
+        # parameter prediction
+        ctrls = self.unit2ctrl(units_frames, f0_frames, phase_frames, volume_frames, spk_id=spk_id, spk_mix_dict=spk_mix_dict)
+        
+        amplitudes_frames = torch.exp(ctrls['amplitudes'])/ 128
+        group_delay = np.pi * torch.tanh(ctrls['group_delay'])
+        noise_param = torch.exp(ctrls['noise_magnitude']) / 128
+        
+        # sinusoids exciter signal 
+        amplitudes_frames = remove_above_fmax(amplitudes_frames, f0_frames, self.sampling_rate / 2, level_start = 1)
+        n_harmonic = amplitudes_frames.shape[-1]
+        level_harmonic = torch.arange(1, n_harmonic + 1).to(phase)
+        sinusoids = 0.
+        for n in range(( n_harmonic - 1) // max_upsample_dim + 1):
+            start = n * max_upsample_dim
+            end = (n + 1) * max_upsample_dim
+            phases = phase * level_harmonic[start:end]
+            amplitudes = upsample(amplitudes_frames[:,:,start:end], self.block_size)
+            sinusoids += (torch.sin(phases) * amplitudes).sum(-1)
+        
+        # harmonic part filter (apply group-delay)
+        harmonic = frequency_filter(
+                        sinusoids,
+                        torch.exp(1.j * torch.cumsum(group_delay, axis = -1)),
+                        hann_window = False)
+                        
+        # noise part filter 
+        noise = torch.rand_like(harmonic) * 2 - 1
+        noise = frequency_filter(
+                        noise,
+                        torch.complex(noise_param, torch.zeros_like(noise_param)),
+                        hann_window = True)
+                        
+        signal = harmonic + noise
+
+        return signal, phase, (harmonic, noise) #, (noise_param, noise_param)
+
+class CombSubFast(torch.nn.Module):
+    def __init__(self, 
+            sampling_rate,
+            block_size,
+            n_unit=256,
+            n_spk=1):
+        super().__init__()
+
+        print(' [DDSP Model] Combtooth Subtractive Synthesiser')
+        # params
+        self.register_buffer("sampling_rate", torch.tensor(sampling_rate))
+        self.register_buffer("block_size", torch.tensor(block_size))
+        self.register_buffer("window", torch.sqrt(torch.hann_window(2 * block_size)))
+        #Unit2Control
+        split_map = {
+            'harmonic_magnitude': block_size + 1, 
+            'harmonic_phase': block_size + 1,
+            'noise_magnitude': block_size + 1
+        }
+        self.unit2ctrl = Unit2Control(n_unit, n_spk, split_map)
+
+    def forward(self, units_frames, f0_frames, volume_frames, spk_id=None, spk_mix_dict=None, initial_phase=None, infer=True, **kwargs):
+        '''
+            units_frames: B x n_frames x n_unit
+            f0_frames: B x n_frames x 1
+            volume_frames: B x n_frames x 1 
+            spk_id: B x 1
+        '''
+        # exciter phase
+        f0 = upsample(f0_frames, self.block_size)
+        if infer:
+            x = torch.cumsum(f0.double() / self.sampling_rate, axis=1)
+        else:
+            x = torch.cumsum(f0 / self.sampling_rate, axis=1)
+        if initial_phase is not None:
+            x += initial_phase.to(x) / 2 / np.pi    
+        x = x - torch.round(x)
+        x = x.to(f0)
+        
+        phase_frames = 2 * np.pi * x[:, ::self.block_size, :]
+        
+        # parameter prediction
+        ctrls = self.unit2ctrl(units_frames, f0_frames, phase_frames, volume_frames, spk_id=spk_id, spk_mix_dict=spk_mix_dict)
+        
+        src_filter = torch.exp(ctrls['harmonic_magnitude'] + 1.j * np.pi * ctrls['harmonic_phase'])
+        src_filter = torch.cat((src_filter, src_filter[:,-1:,:]), 1)
+        noise_filter= torch.exp(ctrls['noise_magnitude']) / 128
+        noise_filter = torch.cat((noise_filter, noise_filter[:,-1:,:]), 1)
+        
+        # combtooth exciter signal 
+        combtooth = torch.sinc(self.sampling_rate * x / (f0 + 1e-3))
+        combtooth = combtooth.squeeze(-1)     
+        combtooth_frames = F.pad(combtooth, (self.block_size, self.block_size)).unfold(1, 2 * self.block_size, self.block_size)
+        combtooth_frames = combtooth_frames * self.window
+        combtooth_fft = torch.fft.rfft(combtooth_frames, 2 * self.block_size)
+        
+        # noise exciter signal
+        noise = torch.rand_like(combtooth) * 2 - 1
+        noise_frames = F.pad(noise, (self.block_size, self.block_size)).unfold(1, 2 * self.block_size, self.block_size)
+        noise_frames = noise_frames * self.window
+        noise_fft = torch.fft.rfft(noise_frames, 2 * self.block_size)
+        
+        # apply the filters 
+        signal_fft = combtooth_fft * src_filter + noise_fft * noise_filter
+
+        # take the ifft to resynthesize audio.
+        signal_frames_out = torch.fft.irfft(signal_fft, 2 * self.block_size) * self.window
+
+        # overlap add
+        fold = torch.nn.Fold(output_size=(1, (signal_frames_out.size(1) + 1) * self.block_size), kernel_size=(1, 2 * self.block_size), stride=(1, self.block_size))
+        signal = fold(signal_frames_out.transpose(1, 2))[:, 0, 0, self.block_size : -self.block_size]
+
+        return signal, phase_frames, (signal, signal)
+
+class CombSub(torch.nn.Module):
+    def __init__(self, 
+            sampling_rate,
+            block_size,
+            n_mag_allpass,
+            n_mag_harmonic,
+            n_mag_noise,
+            n_unit=256,
+            n_spk=1):
+        super().__init__()
+
+        print(' [DDSP Model] Combtooth Subtractive Synthesiser (Old Version)')
+        # params
+        self.register_buffer("sampling_rate", torch.tensor(sampling_rate))
+        self.register_buffer("block_size", torch.tensor(block_size))
+        #Unit2Control
+        split_map = {
+            'group_delay': n_mag_allpass,
+            'harmonic_magnitude': n_mag_harmonic, 
+            'noise_magnitude': n_mag_noise
+        }
+        self.unit2ctrl = Unit2Control(n_unit, n_spk, split_map)
+
+    def forward(self, units_frames, f0_frames, volume_frames, spk_id=None, spk_mix_dict=None, initial_phase=None, infer=True, **kwargs):
+        '''
+            units_frames: B x n_frames x n_unit
+            f0_frames: B x n_frames x 1
+            volume_frames: B x n_frames x 1 
+            spk_id: B x 1
+        '''
+        # exciter phase
+        f0 = upsample(f0_frames, self.block_size)
+        if infer:
+            x = torch.cumsum(f0.double() / self.sampling_rate, axis=1)
+        else:
+            x = torch.cumsum(f0 / self.sampling_rate, axis=1)
+        if initial_phase is not None:
+            x += initial_phase.to(x) / 2 / np.pi    
+        x = x - torch.round(x)
+        x = x.to(f0)
+        
+        phase_frames = 2 * np.pi * x[:, ::self.block_size, :]
+        
+        # parameter prediction
+        ctrls = self.unit2ctrl(units_frames, f0_frames, phase_frames, volume_frames, spk_id=spk_id, spk_mix_dict=spk_mix_dict)
+        
+        group_delay = np.pi * torch.tanh(ctrls['group_delay'])
+        src_param = torch.exp(ctrls['harmonic_magnitude'])
+        noise_param = torch.exp(ctrls['noise_magnitude']) / 128
+        
+        # combtooth exciter signal 
+        combtooth = torch.sinc(self.sampling_rate * x / (f0 + 1e-3))
+        combtooth = combtooth.squeeze(-1)
+        
+        # harmonic part filter (using dynamic-windowed LTV-FIR, with group-delay prediction)
+        harmonic = frequency_filter(
+                        combtooth,
+                        torch.exp(1.j * torch.cumsum(group_delay, axis = -1)),
+                        hann_window = False)
+        harmonic = frequency_filter(
+                        harmonic,
+                        torch.complex(src_param, torch.zeros_like(src_param)),
+                        hann_window = True,
+                        half_width_frames = 1.5 * self.sampling_rate / (f0_frames + 1e-3))
+                  
+        # noise part filter (using constant-windowed LTV-FIR, without group-delay)
+        noise = torch.rand_like(harmonic) * 2 - 1
+        noise = frequency_filter(
+                        noise,
+                        torch.complex(noise_param, torch.zeros_like(noise_param)),
+                        hann_window = True)
+                        
+        signal = harmonic + noise
+
+        return signal, phase_frames, (harmonic, noise)

DDSP-SVC/diffusion/data_loaders.py

@@ -0,0 +1,271 @@
+import os
+import random
+import numpy as np
+import librosa
+import torch
+import random
+from tqdm import tqdm
+from torch.utils.data import Dataset
+
+def traverse_dir(
+        root_dir,
+        extension,
+        amount=None,
+        str_include=None,
+        str_exclude=None,
+        is_pure=False,
+        is_sort=False,
+        is_ext=True):
+
+    file_list = []
+    cnt = 0
+    for root, _, files in os.walk(root_dir):
+        for file in files:
+            if file.endswith(extension):
+                # path
+                mix_path = os.path.join(root, file)
+                pure_path = mix_path[len(root_dir)+1:] if is_pure else mix_path
+
+                # amount
+                if (amount is not None) and (cnt == amount):
+                    if is_sort:
+                        file_list.sort()
+                    return file_list
+                
+                # check string
+                if (str_include is not None) and (str_include not in pure_path):
+                    continue
+                if (str_exclude is not None) and (str_exclude in pure_path):
+                    continue
+                
+                if not is_ext:
+                    ext = pure_path.split('.')[-1]
+                    pure_path = pure_path[:-(len(ext)+1)]
+                file_list.append(pure_path)
+                cnt += 1
+    if is_sort:
+        file_list.sort()
+    return file_list
+
+
+def get_data_loaders(args, whole_audio=False):
+    data_train = AudioDataset(
+        args.data.train_path,
+        waveform_sec=args.data.duration,
+        hop_size=args.data.block_size,
+        sample_rate=args.data.sampling_rate,
+        load_all_data=args.train.cache_all_data,
+        whole_audio=whole_audio,
+        n_spk=args.model.n_spk,
+        device=args.train.cache_device,
+        fp16=args.train.cache_fp16,
+        use_aug=True)
+    loader_train = torch.utils.data.DataLoader(
+        data_train ,
+        batch_size=args.train.batch_size if not whole_audio else 1,
+        shuffle=True,
+        num_workers=args.train.num_workers if args.train.cache_device=='cpu' else 0,
+        persistent_workers=(args.train.num_workers > 0) if args.train.cache_device=='cpu' else False,
+        pin_memory=True if args.train.cache_device=='cpu' else False
+    )
+    data_valid = AudioDataset(
+        args.data.valid_path,
+        waveform_sec=args.data.duration,
+        hop_size=args.data.block_size,
+        sample_rate=args.data.sampling_rate,
+        load_all_data=args.train.cache_all_data,
+        whole_audio=True,
+        n_spk=args.model.n_spk)
+    loader_valid = torch.utils.data.DataLoader(
+        data_valid,
+        batch_size=1,
+        shuffle=False,
+        num_workers=0,
+        pin_memory=True
+    )
+    return loader_train, loader_valid 
+
+
+class AudioDataset(Dataset):
+    def __init__(
+        self,
+        path_root,
+        waveform_sec,
+        hop_size,
+        sample_rate,
+        load_all_data=True,
+        whole_audio=False,
+        n_spk=1,
+        device='cpu',
+        fp16=False,
+        use_aug=False,
+    ):
+        super().__init__()
+        
+        self.waveform_sec = waveform_sec
+        self.sample_rate = sample_rate
+        self.hop_size = hop_size
+        self.path_root = path_root
+        self.paths = traverse_dir(
+            os.path.join(path_root, 'audio'),
+            extension='wav',
+            is_pure=True,
+            is_sort=True,
+            is_ext=False
+        )
+        self.whole_audio = whole_audio
+        self.use_aug = use_aug
+        self.data_buffer={}
+        self.pitch_aug_dict = np.load(os.path.join(self.path_root, 'pitch_aug_dict.npy'), allow_pickle=True).item()
+        if load_all_data:
+            print('Load all the data from :', path_root)
+        else:
+            print('Load the f0, volume data from :', path_root)
+        for name in tqdm(self.paths, total=len(self.paths)):
+            path_audio = os.path.join(self.path_root, 'audio', name) + '.wav'
+            duration = librosa.get_duration(filename = path_audio, sr = self.sample_rate)
+            
+            path_f0 = os.path.join(self.path_root, 'f0', name) + '.npy'
+            f0 = np.load(path_f0)
+            f0 = torch.from_numpy(f0).float().unsqueeze(-1).to(device)
+                
+            path_volume = os.path.join(self.path_root, 'volume', name) + '.npy'
+            volume = np.load(path_volume)
+            volume = torch.from_numpy(volume).float().unsqueeze(-1).to(device)
+            
+            path_augvol = os.path.join(self.path_root, 'aug_vol', name) + '.npy'
+            aug_vol = np.load(path_augvol)
+            aug_vol = torch.from_numpy(aug_vol).float().unsqueeze(-1).to(device)
+                        
+            if n_spk is not None and n_spk > 1:
+                spk_id = int(os.path.dirname(name)) if str.isdigit(os.path.dirname(name)) else 0
+                if spk_id < 1 or spk_id > n_spk:
+                    raise ValueError(' [x] Muiti-speaker traing error : spk_id must be a positive integer from 1 to n_spk ')
+            else:
+                spk_id = 1
+            spk_id = torch.LongTensor(np.array([spk_id])).to(device)
+
+            if load_all_data:
+                '''
+                audio, sr = librosa.load(path_audio, sr=self.sample_rate)
+                if len(audio.shape) > 1:
+                    audio = librosa.to_mono(audio)
+                audio = torch.from_numpy(audio).to(device)
+                '''
+                path_mel = os.path.join(self.path_root, 'mel', name) + '.npy'
+                mel = np.load(path_mel)
+                mel = torch.from_numpy(mel).to(device)
+                
+                path_augmel = os.path.join(self.path_root, 'aug_mel', name) + '.npy'
+                aug_mel = np.load(path_augmel)
+                aug_mel = torch.from_numpy(aug_mel).to(device)
+                
+                path_units = os.path.join(self.path_root, 'units', name) + '.npy'
+                units = np.load(path_units)
+                units = torch.from_numpy(units).to(device)
+                
+                if fp16:
+                    mel = mel.half()
+                    aug_mel = aug_mel.half()
+                    units = units.half()
+                    
+                self.data_buffer[name] = {
+                        'duration': duration,
+                        'mel': mel,
+                        'aug_mel': aug_mel,
+                        'units': units,
+                        'f0': f0,
+                        'volume': volume,
+                        'aug_vol': aug_vol,
+                        'spk_id': spk_id
+                        }
+            else:
+                self.data_buffer[name] = {
+                        'duration': duration,
+                        'f0': f0,
+                        'volume': volume,
+                        'aug_vol': aug_vol,
+                        'spk_id': spk_id
+                        }
+           
+
+    def __getitem__(self, file_idx):
+        name = self.paths[file_idx]
+        data_buffer = self.data_buffer[name]
+        # check duration. if too short, then skip
+        if data_buffer['duration'] < (self.waveform_sec + 0.1):
+            return self.__getitem__( (file_idx + 1) % len(self.paths))
+            
+        # get item
+        return self.get_data(name, data_buffer)
+
+    def get_data(self, name, data_buffer):
+        frame_resolution = self.hop_size / self.sample_rate
+        duration = data_buffer['duration']
+        waveform_sec = duration if self.whole_audio else self.waveform_sec
+        
+        # load audio
+        idx_from = 0 if self.whole_audio else random.uniform(0, duration - waveform_sec - 0.1)
+        start_frame = int(idx_from / frame_resolution)
+        units_frame_len = int(waveform_sec / frame_resolution)
+        aug_flag = random.choice([True, False]) and self.use_aug
+        '''
+        audio = data_buffer.get('audio')
+        if audio is None:
+            path_audio = os.path.join(self.path_root, 'audio', name) + '.wav'
+            audio, sr = librosa.load(
+                    path_audio, 
+                    sr = self.sample_rate, 
+                    offset = start_frame * frame_resolution,
+                    duration = waveform_sec)
+            if len(audio.shape) > 1:
+                audio = librosa.to_mono(audio)
+            # clip audio into N seconds
+            audio = audio[ : audio.shape[-1] // self.hop_size * self.hop_size]       
+            audio = torch.from_numpy(audio).float()
+        else:
+            audio = audio[start_frame * self.hop_size : (start_frame + units_frame_len) * self.hop_size]
+        '''
+        # load mel
+        mel_key = 'aug_mel' if aug_flag else 'mel'
+        mel = data_buffer.get(mel_key)
+        if mel is None:
+            mel = os.path.join(self.path_root, mel_key, name) + '.npy'
+            mel = np.load(mel)
+            mel = mel[start_frame : start_frame + units_frame_len]
+            mel = torch.from_numpy(mel).float() 
+        else:
+            mel = mel[start_frame : start_frame + units_frame_len]
+            
+        # load units
+        units = data_buffer.get('units')
+        if units is None:
+            units = os.path.join(self.path_root, 'units', name) + '.npy'
+            units = np.load(units)
+            units = units[start_frame : start_frame + units_frame_len]
+            units = torch.from_numpy(units).float() 
+        else:
+            units = units[start_frame : start_frame + units_frame_len]
+
+        # load f0
+        f0 = data_buffer.get('f0')
+        aug_shift = 0
+        if aug_flag:
+            aug_shift = self.pitch_aug_dict[name]
+        f0_frames = 2 ** (aug_shift / 12) * f0[start_frame : start_frame + units_frame_len]
+        
+        # load volume
+        vol_key = 'aug_vol' if aug_flag else 'volume'
+        volume = data_buffer.get(vol_key)
+        volume_frames = volume[start_frame : start_frame + units_frame_len]
+        
+        # load spk_id
+        spk_id = data_buffer.get('spk_id')
+        
+        # load shift
+        aug_shift = torch.LongTensor(np.array([[aug_shift]]))
+        
+        return dict(mel=mel, f0=f0_frames, volume=volume_frames, units=units, spk_id=spk_id, aug_shift=aug_shift, name=name)
+
+    def __len__(self):
+        return len(self.paths)

DDSP-SVC/diffusion/diffusion.py

@@ -0,0 +1,317 @@
+from collections import deque
+from functools import partial
+from inspect import isfunction
+import torch.nn.functional as F
+import librosa.sequence
+import numpy as np
+import torch
+from torch import nn
+from tqdm import tqdm
+
+
+def exists(x):
+    return x is not None
+
+
+def default(val, d):
+    if exists(val):
+        return val
+    return d() if isfunction(d) else d
+
+
+def extract(a, t, x_shape):
+    b, *_ = t.shape
+    out = a.gather(-1, t)
+    return out.reshape(b, *((1,) * (len(x_shape) - 1)))
+
+
+def noise_like(shape, device, repeat=False):
+    repeat_noise = lambda: torch.randn((1, *shape[1:]), device=device).repeat(shape[0], *((1,) * (len(shape) - 1)))
+    noise = lambda: torch.randn(shape, device=device)
+    return repeat_noise() if repeat else noise()
+
+
+def linear_beta_schedule(timesteps, max_beta=0.02):
+    """
+    linear schedule
+    """
+    betas = np.linspace(1e-4, max_beta, timesteps)
+    return betas
+
+
+def cosine_beta_schedule(timesteps, s=0.008):
+    """
+    cosine schedule
+    as proposed in https://openreview.net/forum?id=-NEXDKk8gZ
+    """
+    steps = timesteps + 1
+    x = np.linspace(0, steps, steps)
+    alphas_cumprod = np.cos(((x / steps) + s) / (1 + s) * np.pi * 0.5) ** 2
+    alphas_cumprod = alphas_cumprod / alphas_cumprod[0]
+    betas = 1 - (alphas_cumprod[1:] / alphas_cumprod[:-1])
+    return np.clip(betas, a_min=0, a_max=0.999)
+
+
+beta_schedule = {
+    "cosine": cosine_beta_schedule,
+    "linear": linear_beta_schedule,
+}
+
+
+class GaussianDiffusion(nn.Module):
+    def __init__(self, 
+                denoise_fn, 
+                out_dims=128,
+                timesteps=1000, 
+                k_step=1000,
+                max_beta=0.02,
+                spec_min=-12, 
+                spec_max=2):
+        super().__init__()
+        self.denoise_fn = denoise_fn
+        self.out_dims = out_dims
+        betas = beta_schedule['linear'](timesteps, max_beta=max_beta)
+
+        alphas = 1. - betas
+        alphas_cumprod = np.cumprod(alphas, axis=0)
+        alphas_cumprod_prev = np.append(1., alphas_cumprod[:-1])
+
+        timesteps, = betas.shape
+        self.num_timesteps = int(timesteps)
+        self.k_step = k_step
+
+        self.noise_list = deque(maxlen=4)
+
+        to_torch = partial(torch.tensor, dtype=torch.float32)
+
+        self.register_buffer('betas', to_torch(betas))
+        self.register_buffer('alphas_cumprod', to_torch(alphas_cumprod))
+        self.register_buffer('alphas_cumprod_prev', to_torch(alphas_cumprod_prev))
+
+        # calculations for diffusion q(x_t | x_{t-1}) and others
+        self.register_buffer('sqrt_alphas_cumprod', to_torch(np.sqrt(alphas_cumprod)))
+        self.register_buffer('sqrt_one_minus_alphas_cumprod', to_torch(np.sqrt(1. - alphas_cumprod)))
+        self.register_buffer('log_one_minus_alphas_cumprod', to_torch(np.log(1. - alphas_cumprod)))
+        self.register_buffer('sqrt_recip_alphas_cumprod', to_torch(np.sqrt(1. / alphas_cumprod)))
+        self.register_buffer('sqrt_recipm1_alphas_cumprod', to_torch(np.sqrt(1. / alphas_cumprod - 1)))
+
+        # calculations for posterior q(x_{t-1} | x_t, x_0)
+        posterior_variance = betas * (1. - alphas_cumprod_prev) / (1. - alphas_cumprod)
+        # above: equal to 1. / (1. / (1. - alpha_cumprod_tm1) + alpha_t / beta_t)
+        self.register_buffer('posterior_variance', to_torch(posterior_variance))
+        # below: log calculation clipped because the posterior variance is 0 at the beginning of the diffusion chain
+        self.register_buffer('posterior_log_variance_clipped', to_torch(np.log(np.maximum(posterior_variance, 1e-20))))
+        self.register_buffer('posterior_mean_coef1', to_torch(
+            betas * np.sqrt(alphas_cumprod_prev) / (1. - alphas_cumprod)))
+        self.register_buffer('posterior_mean_coef2', to_torch(
+            (1. - alphas_cumprod_prev) * np.sqrt(alphas) / (1. - alphas_cumprod)))
+
+        self.register_buffer('spec_min', torch.FloatTensor([spec_min])[None, None, :out_dims])
+        self.register_buffer('spec_max', torch.FloatTensor([spec_max])[None, None, :out_dims])
+
+    def q_mean_variance(self, x_start, t):
+        mean = extract(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start
+        variance = extract(1. - self.alphas_cumprod, t, x_start.shape)
+        log_variance = extract(self.log_one_minus_alphas_cumprod, t, x_start.shape)
+        return mean, variance, log_variance
+
+    def predict_start_from_noise(self, x_t, t, noise):
+        return (
+                extract(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t -
+                extract(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) * noise
+        )
+
+    def q_posterior(self, x_start, x_t, t):
+        posterior_mean = (
+                extract(self.posterior_mean_coef1, t, x_t.shape) * x_start +
+                extract(self.posterior_mean_coef2, t, x_t.shape) * x_t
+        )
+        posterior_variance = extract(self.posterior_variance, t, x_t.shape)
+        posterior_log_variance_clipped = extract(self.posterior_log_variance_clipped, t, x_t.shape)
+        return posterior_mean, posterior_variance, posterior_log_variance_clipped
+
+    def p_mean_variance(self, x, t, cond):
+        noise_pred = self.denoise_fn(x, t, cond=cond)
+        x_recon = self.predict_start_from_noise(x, t=t, noise=noise_pred)
+
+        x_recon.clamp_(-1., 1.)
+
+        model_mean, posterior_variance, posterior_log_variance = self.q_posterior(x_start=x_recon, x_t=x, t=t)
+        return model_mean, posterior_variance, posterior_log_variance
+
+    @torch.no_grad()
+    def p_sample(self, x, t, cond, clip_denoised=True, repeat_noise=False):
+        b, *_, device = *x.shape, x.device
+        model_mean, _, model_log_variance = self.p_mean_variance(x=x, t=t, cond=cond)
+        noise = noise_like(x.shape, device, repeat_noise)
+        # no noise when t == 0
+        nonzero_mask = (1 - (t == 0).float()).reshape(b, *((1,) * (len(x.shape) - 1)))
+        return model_mean + nonzero_mask * (0.5 * model_log_variance).exp() * noise
+
+    @torch.no_grad()
+    def p_sample_plms(self, x, t, interval, cond, clip_denoised=True, repeat_noise=False):
+        """
+        Use the PLMS method from
+        [Pseudo Numerical Methods for Diffusion Models on Manifolds](https://arxiv.org/abs/2202.09778).
+        """
+
+        def get_x_pred(x, noise_t, t):
+            a_t = extract(self.alphas_cumprod, t, x.shape)
+            a_prev = extract(self.alphas_cumprod, torch.max(t - interval, torch.zeros_like(t)), x.shape)
+            a_t_sq, a_prev_sq = a_t.sqrt(), a_prev.sqrt()
+
+            x_delta = (a_prev - a_t) * ((1 / (a_t_sq * (a_t_sq + a_prev_sq))) * x - 1 / (
+                    a_t_sq * (((1 - a_prev) * a_t).sqrt() + ((1 - a_t) * a_prev).sqrt())) * noise_t)
+            x_pred = x + x_delta
+
+            return x_pred
+
+        noise_list = self.noise_list
+        noise_pred = self.denoise_fn(x, t, cond=cond)
+
+        if len(noise_list) == 0:
+            x_pred = get_x_pred(x, noise_pred, t)
+            noise_pred_prev = self.denoise_fn(x_pred, max(t - interval, 0), cond=cond)
+            noise_pred_prime = (noise_pred + noise_pred_prev) / 2
+        elif len(noise_list) == 1:
+            noise_pred_prime = (3 * noise_pred - noise_list[-1]) / 2
+        elif len(noise_list) == 2:
+            noise_pred_prime = (23 * noise_pred - 16 * noise_list[-1] + 5 * noise_list[-2]) / 12
+        else:
+            noise_pred_prime = (55 * noise_pred - 59 * noise_list[-1] + 37 * noise_list[-2] - 9 * noise_list[-3]) / 24
+
+        x_prev = get_x_pred(x, noise_pred_prime, t)
+        noise_list.append(noise_pred)
+
+        return x_prev
+
+    def q_sample(self, x_start, t, noise=None):
+        noise = default(noise, lambda: torch.randn_like(x_start))
+        return (
+                extract(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start +
+                extract(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) * noise
+        )
+
+    def p_losses(self, x_start, t, cond, noise=None, loss_type='l2'):
+        noise = default(noise, lambda: torch.randn_like(x_start))
+
+        x_noisy = self.q_sample(x_start=x_start, t=t, noise=noise)
+        x_recon = self.denoise_fn(x_noisy, t, cond)
+
+        if loss_type == 'l1':
+            loss = (noise - x_recon).abs().mean()
+        elif loss_type == 'l2':
+            loss = F.mse_loss(noise, x_recon)
+        else:
+            raise NotImplementedError()
+
+        return loss
+
+    def forward(self, 
+                condition, 
+                gt_spec=None, 
+                infer=True, 
+                infer_speedup=10, 
+                method='dpm-solver',
+                k_step=300,
+                use_tqdm=True):
+        """
+            conditioning diffusion, use fastspeech2 encoder output as the condition
+        """
+        cond = condition.transpose(1, 2)
+        b, device = condition.shape[0], condition.device
+
+        if not infer:
+            spec = self.norm_spec(gt_spec)
+            t = torch.randint(0, self.k_step, (b,), device=device).long()
+            norm_spec = spec.transpose(1, 2)[:, None, :, :]  # [B, 1, M, T]
+            return self.p_losses(norm_spec, t, cond=cond)
+        else:
+            shape = (cond.shape[0], 1, self.out_dims, cond.shape[2])
+            
+            if gt_spec is None:
+                t = self.k_step
+                x = torch.randn(shape, device=device)
+            else:
+                t = k_step
+                norm_spec = self.norm_spec(gt_spec)
+                norm_spec = norm_spec.transpose(1, 2)[:, None, :, :]
+                x = self.q_sample(x_start=norm_spec, t=torch.tensor([t - 1], device=device).long())
+                        
+            if method is not None and infer_speedup > 1:
+                if method == 'dpm-solver':
+                    from .dpm_solver_pytorch import NoiseScheduleVP, model_wrapper, DPM_Solver
+                    # 1. Define the noise schedule.
+                    noise_schedule = NoiseScheduleVP(schedule='discrete', betas=self.betas[:t])
+
+                    # 2. Convert your discrete-time `model` to the continuous-time
+                    # noise prediction model. Here is an example for a diffusion model
+                    # `model` with the noise prediction type ("noise") .
+                    def my_wrapper(fn):
+                        def wrapped(x, t, **kwargs):
+                            ret = fn(x, t, **kwargs)
+                            if use_tqdm:
+                                self.bar.update(1)
+                            return ret
+
+                        return wrapped
+
+                    model_fn = model_wrapper(
+                        my_wrapper(self.denoise_fn),
+                        noise_schedule,
+                        model_type="noise",  # or "x_start" or "v" or "score"
+                        model_kwargs={"cond": cond}
+                    )
+
+                    # 3. Define dpm-solver and sample by singlestep DPM-Solver.
+                    # (We recommend singlestep DPM-Solver for unconditional sampling)
+                    # You can adjust the `steps` to balance the computation
+                    # costs and the sample quality.
+                    dpm_solver = DPM_Solver(model_fn, noise_schedule)
+
+                    steps = t // infer_speedup
+                    if use_tqdm:
+                        self.bar = tqdm(desc="sample time step", total=steps)
+                    x = dpm_solver.sample(
+                        x,
+                        steps=steps,
+                        order=3,
+                        skip_type="time_uniform",
+                        method="singlestep",
+                    )
+                    if use_tqdm:
+                        self.bar.close()
+                elif method == 'pndm':
+                    self.noise_list = deque(maxlen=4)
+                    if use_tqdm:
+                        for i in tqdm(
+                                reversed(range(0, t, infer_speedup)), desc='sample time step',
+                                total=t // infer_speedup,
+                        ):
+                            x = self.p_sample_plms(
+                                x, torch.full((b,), i, device=device, dtype=torch.long),
+                                infer_speedup, cond=cond
+                            )
+                    else:
+                        for i in reversed(range(0, t, infer_speedup)):
+                            x = self.p_sample_plms(
+                                x, torch.full((b,), i, device=device, dtype=torch.long),
+                                infer_speedup, cond=cond
+                            )
+                else:
+                    raise NotImplementedError(method)
+            else:
+                if use_tqdm:
+                    for i in tqdm(reversed(range(0, t)), desc='sample time step', total=t):
+                        x = self.p_sample(x, torch.full((b,), i, device=device, dtype=torch.long), cond)
+                else:
+                    for i in reversed(range(0, t)):
+                        x = self.p_sample(x, torch.full((b,), i, device=device, dtype=torch.long), cond)
+            x = x.squeeze(1).transpose(1, 2)  # [B, T, M]
+            return self.denorm_spec(x)
+
+    def norm_spec(self, x):
+        return (x - self.spec_min) / (self.spec_max - self.spec_min) * 2 - 1
+
+    def denorm_spec(self, x):
+        return (x + 1) / 2 * (self.spec_max - self.spec_min) + self.spec_min

DDSP-SVC/diffusion/dpm_solver_pytorch.py

@@ -0,0 +1,1201 @@
+import math
+
+import torch
+
+
+class NoiseScheduleVP:
+    def __init__(
+            self,
+            schedule='discrete',
+            betas=None,
+            alphas_cumprod=None,
+            continuous_beta_0=0.1,
+            continuous_beta_1=20.,
+    ):
+        """Create a wrapper class for the forward SDE (VP type).
+
+        ***
+        Update: We support discrete-time diffusion models by implementing a picewise linear interpolation for log_alpha_t.
+                We recommend to use schedule='discrete' for the discrete-time diffusion models, especially for high-resolution images.
+        ***
+
+        The forward SDE ensures that the condition distribution q_{t|0}(x_t | x_0) = N ( alpha_t * x_0, sigma_t^2 * I ).
+        We further define lambda_t = log(alpha_t) - log(sigma_t), which is the half-logSNR (described in the DPM-Solver paper).
+        Therefore, we implement the functions for computing alpha_t, sigma_t and lambda_t. For t in [0, T], we have:
+
+            log_alpha_t = self.marginal_log_mean_coeff(t)
+            sigma_t = self.marginal_std(t)
+            lambda_t = self.marginal_lambda(t)
+
+        Moreover, as lambda(t) is an invertible function, we also support its inverse function:
+
+            t = self.inverse_lambda(lambda_t)
+
+        ===============================================================
+
+        We support both discrete-time DPMs (trained on n = 0, 1, ..., N-1) and continuous-time DPMs (trained on t in [t_0, T]).
+
+        1. For discrete-time DPMs:
+
+            For discrete-time DPMs trained on n = 0, 1, ..., N-1, we convert the discrete steps to continuous time steps by:
+                t_i = (i + 1) / N
+            e.g. for N = 1000, we have t_0 = 1e-3 and T = t_{N-1} = 1.
+            We solve the corresponding diffusion ODE from time T = 1 to time t_0 = 1e-3.
+
+            Args:
+                betas: A `torch.Tensor`. The beta array for the discrete-time DPM. (See the original DDPM paper for details)
+                alphas_cumprod: A `torch.Tensor`. The cumprod alphas for the discrete-time DPM. (See the original DDPM paper for details)
+
+            Note that we always have alphas_cumprod = cumprod(betas). Therefore, we only need to set one of `betas` and `alphas_cumprod`.
+
+            **Important**:  Please pay special attention for the args for `alphas_cumprod`:
+                The `alphas_cumprod` is the \hat{alpha_n} arrays in the notations of DDPM. Specifically, DDPMs assume that
+                    q_{t_n | 0}(x_{t_n} | x_0) = N ( \sqrt{\hat{alpha_n}} * x_0, (1 - \hat{alpha_n}) * I ).
+                Therefore, the notation \hat{alpha_n} is different from the notation alpha_t in DPM-Solver. In fact, we have
+                    alpha_{t_n} = \sqrt{\hat{alpha_n}},
+                and
+                    log(alpha_{t_n}) = 0.5 * log(\hat{alpha_n}).
+
+
+        2. For continuous-time DPMs:
+
+            We support two types of VPSDEs: linear (DDPM) and cosine (improved-DDPM). The hyperparameters for the noise
+            schedule are the default settings in DDPM and improved-DDPM:
+
+            Args:
+                beta_min: A `float` number. The smallest beta for the linear schedule.
+                beta_max: A `float` number. The largest beta for the linear schedule.
+                cosine_s: A `float` number. The hyperparameter in the cosine schedule.
+                cosine_beta_max: A `float` number. The hyperparameter in the cosine schedule.
+                T: A `float` number. The ending time of the forward process.
+
+        ===============================================================
+
+        Args:
+            schedule: A `str`. The noise schedule of the forward SDE. 'discrete' for discrete-time DPMs,
+                    'linear' or 'cosine' for continuous-time DPMs.
+        Returns:
+            A wrapper object of the forward SDE (VP type).
+        
+        ===============================================================
+
+        Example:
+
+        # For discrete-time DPMs, given betas (the beta array for n = 0, 1, ..., N - 1):
+        >>> ns = NoiseScheduleVP('discrete', betas=betas)
+
+        # For discrete-time DPMs, given alphas_cumprod (the \hat{alpha_n} array for n = 0, 1, ..., N - 1):
+        >>> ns = NoiseScheduleVP('discrete', alphas_cumprod=alphas_cumprod)
+
+        # For continuous-time DPMs (VPSDE), linear schedule:
+        >>> ns = NoiseScheduleVP('linear', continuous_beta_0=0.1, continuous_beta_1=20.)
+
+        """
+
+        if schedule not in ['discrete', 'linear', 'cosine']:
+            raise ValueError(
+                "Unsupported noise schedule {}. The schedule needs to be 'discrete' or 'linear' or 'cosine'".format(
+                    schedule))
+
+        self.schedule = schedule
+        if schedule == 'discrete':
+            if betas is not None:
+                log_alphas = 0.5 * torch.log(1 - betas).cumsum(dim=0)
+            else:
+                assert alphas_cumprod is not None
+                log_alphas = 0.5 * torch.log(alphas_cumprod)
+            self.total_N = len(log_alphas)
+            self.T = 1.
+            self.t_array = torch.linspace(0., 1., self.total_N + 1)[1:].reshape((1, -1))
+            self.log_alpha_array = log_alphas.reshape((1, -1,))
+        else:
+            self.total_N = 1000
+            self.beta_0 = continuous_beta_0
+            self.beta_1 = continuous_beta_1
+            self.cosine_s = 0.008
+            self.cosine_beta_max = 999.
+            self.cosine_t_max = math.atan(self.cosine_beta_max * (1. + self.cosine_s) / math.pi) * 2. * (
+                        1. + self.cosine_s) / math.pi - self.cosine_s
+            self.cosine_log_alpha_0 = math.log(math.cos(self.cosine_s / (1. + self.cosine_s) * math.pi / 2.))
+            self.schedule = schedule
+            if schedule == 'cosine':
+                # For the cosine schedule, T = 1 will have numerical issues. So we manually set the ending time T.
+                # Note that T = 0.9946 may be not the optimal setting. However, we find it works well.
+                self.T = 0.9946
+            else:
+                self.T = 1.
+
+    def marginal_log_mean_coeff(self, t):
+        """
+        Compute log(alpha_t) of a given continuous-time label t in [0, T].
+        """
+        if self.schedule == 'discrete':
+            return interpolate_fn(t.reshape((-1, 1)), self.t_array.to(t.device),
+                                  self.log_alpha_array.to(t.device)).reshape((-1))
+        elif self.schedule == 'linear':
+            return -0.25 * t ** 2 * (self.beta_1 - self.beta_0) - 0.5 * t * self.beta_0
+        elif self.schedule == 'cosine':
+            log_alpha_fn = lambda s: torch.log(torch.cos((s + self.cosine_s) / (1. + self.cosine_s) * math.pi / 2.))
+            log_alpha_t = log_alpha_fn(t) - self.cosine_log_alpha_0
+            return log_alpha_t
+
+    def marginal_alpha(self, t):
+        """
+        Compute alpha_t of a given continuous-time label t in [0, T].
+        """
+        return torch.exp(self.marginal_log_mean_coeff(t))
+
+    def marginal_std(self, t):
+        """
+        Compute sigma_t of a given continuous-time label t in [0, T].
+        """
+        return torch.sqrt(1. - torch.exp(2. * self.marginal_log_mean_coeff(t)))
+
+    def marginal_lambda(self, t):
+        """
+        Compute lambda_t = log(alpha_t) - log(sigma_t) of a given continuous-time label t in [0, T].
+        """
+        log_mean_coeff = self.marginal_log_mean_coeff(t)
+        log_std = 0.5 * torch.log(1. - torch.exp(2. * log_mean_coeff))
+        return log_mean_coeff - log_std
+
+    def inverse_lambda(self, lamb):
+        """
+        Compute the continuous-time label t in [0, T] of a given half-logSNR lambda_t.
+        """
+        if self.schedule == 'linear':
+            tmp = 2. * (self.beta_1 - self.beta_0) * torch.logaddexp(-2. * lamb, torch.zeros((1,)).to(lamb))
+            Delta = self.beta_0 ** 2 + tmp
+            return tmp / (torch.sqrt(Delta) + self.beta_0) / (self.beta_1 - self.beta_0)
+        elif self.schedule == 'discrete':
+            log_alpha = -0.5 * torch.logaddexp(torch.zeros((1,)).to(lamb.device), -2. * lamb)
+            t = interpolate_fn(log_alpha.reshape((-1, 1)), torch.flip(self.log_alpha_array.to(lamb.device), [1]),
+                               torch.flip(self.t_array.to(lamb.device), [1]))
+            return t.reshape((-1,))
+        else:
+            log_alpha = -0.5 * torch.logaddexp(-2. * lamb, torch.zeros((1,)).to(lamb))
+            t_fn = lambda log_alpha_t: torch.arccos(torch.exp(log_alpha_t + self.cosine_log_alpha_0)) * 2. * (
+                        1. + self.cosine_s) / math.pi - self.cosine_s
+            t = t_fn(log_alpha)
+            return t
+
+
+def model_wrapper(
+        model,
+        noise_schedule,
+        model_type="noise",
+        model_kwargs={},
+        guidance_type="uncond",
+        condition=None,
+        unconditional_condition=None,
+        guidance_scale=1.,
+        classifier_fn=None,
+        classifier_kwargs={},
+):
+    """Create a wrapper function for the noise prediction model.
+
+    DPM-Solver needs to solve the continuous-time diffusion ODEs. For DPMs trained on discrete-time labels, we need to
+    firstly wrap the model function to a noise prediction model that accepts the continuous time as the input.
+
+    We support four types of the diffusion model by setting `model_type`:
+
+        1. "noise": noise prediction model. (Trained by predicting noise).
+
+        2. "x_start": data prediction model. (Trained by predicting the data x_0 at time 0).
+
+        3. "v": velocity prediction model. (Trained by predicting the velocity).
+            The "v" prediction is derivation detailed in Appendix D of [1], and is used in Imagen-Video [2].
+
+            [1] Salimans, Tim, and Jonathan Ho. "Progressive distillation for fast sampling of diffusion models."
+                arXiv preprint arXiv:2202.00512 (2022).
+            [2] Ho, Jonathan, et al. "Imagen Video: High Definition Video Generation with Diffusion Models."
+                arXiv preprint arXiv:2210.02303 (2022).
+    
+        4. "score": marginal score function. (Trained by denoising score matching).
+            Note that the score function and the noise prediction model follows a simple relationship:
+            ```
+                noise(x_t, t) = -sigma_t * score(x_t, t)
+            ```
+
+    We support three types of guided sampling by DPMs by setting `guidance_type`:
+        1. "uncond": unconditional sampling by DPMs.
+            The input `model` has the following format:
+            ``
+                model(x, t_input, **model_kwargs) -> noise | x_start | v | score
+            ``
+
+        2. "classifier": classifier guidance sampling [3] by DPMs and another classifier.
+            The input `model` has the following format:
+            ``
+                model(x, t_input, **model_kwargs) -> noise | x_start | v | score
+            `` 
+
+            The input `classifier_fn` has the following format:
+            ``
+                classifier_fn(x, t_input, cond, **classifier_kwargs) -> logits(x, t_input, cond)
+            ``
+
+            [3] P. Dhariwal and A. Q. Nichol, "Diffusion models beat GANs on image synthesis,"
+                in Advances in Neural Information Processing Systems, vol. 34, 2021, pp. 8780-8794.
+
+        3. "classifier-free": classifier-free guidance sampling by conditional DPMs.
+            The input `model` has the following format:
+            ``
+                model(x, t_input, cond, **model_kwargs) -> noise | x_start | v | score
+            `` 
+            And if cond == `unconditional_condition`, the model output is the unconditional DPM output.
+
+            [4] Ho, Jonathan, and Tim Salimans. "Classifier-free diffusion guidance."
+                arXiv preprint arXiv:2207.12598 (2022).
+        
+
+    The `t_input` is the time label of the model, which may be discrete-time labels (i.e. 0 to 999)
+    or continuous-time labels (i.e. epsilon to T).
+
+    We wrap the model function to accept only `x` and `t_continuous` as inputs, and outputs the predicted noise:
+    ``
+        def model_fn(x, t_continuous) -> noise:
+            t_input = get_model_input_time(t_continuous)
+            return noise_pred(model, x, t_input, **model_kwargs)         
+    ``
+    where `t_continuous` is the continuous time labels (i.e. epsilon to T). And we use `model_fn` for DPM-Solver.
+
+    ===============================================================
+
+    Args:
+        model: A diffusion model with the corresponding format described above.
+        noise_schedule: A noise schedule object, such as NoiseScheduleVP.
+        model_type: A `str`. The parameterization type of the diffusion model.
+                    "noise" or "x_start" or "v" or "score".
+        model_kwargs: A `dict`. A dict for the other inputs of the model function.
+        guidance_type: A `str`. The type of the guidance for sampling.
+                    "uncond" or "classifier" or "classifier-free".
+        condition: A pytorch tensor. The condition for the guided sampling.
+                    Only used for "classifier" or "classifier-free" guidance type.
+        unconditional_condition: A pytorch tensor. The condition for the unconditional sampling.
+                    Only used for "classifier-free" guidance type.
+        guidance_scale: A `float`. The scale for the guided sampling.
+        classifier_fn: A classifier function. Only used for the classifier guidance.
+        classifier_kwargs: A `dict`. A dict for the other inputs of the classifier function.
+    Returns:
+        A noise prediction model that accepts the noised data and the continuous time as the inputs.
+    """
+
+    def get_model_input_time(t_continuous):
+        """
+        Convert the continuous-time `t_continuous` (in [epsilon, T]) to the model input time.
+        For discrete-time DPMs, we convert `t_continuous` in [1 / N, 1] to `t_input` in [0, 1000 * (N - 1) / N].
+        For continuous-time DPMs, we just use `t_continuous`.
+        """
+        if noise_schedule.schedule == 'discrete':
+            return (t_continuous - 1. / noise_schedule.total_N) * noise_schedule.total_N
+        else:
+            return t_continuous
+
+    def noise_pred_fn(x, t_continuous, cond=None):
+        if t_continuous.reshape((-1,)).shape[0] == 1:
+            t_continuous = t_continuous.expand((x.shape[0]))
+        t_input = get_model_input_time(t_continuous)
+        if cond is None:
+            output = model(x, t_input, **model_kwargs)
+        else:
+            output = model(x, t_input, cond, **model_kwargs)
+        if model_type == "noise":
+            return output
+        elif model_type == "x_start":
+            alpha_t, sigma_t = noise_schedule.marginal_alpha(t_continuous), noise_schedule.marginal_std(t_continuous)
+            dims = x.dim()
+            return (x - expand_dims(alpha_t, dims) * output) / expand_dims(sigma_t, dims)
+        elif model_type == "v":
+            alpha_t, sigma_t = noise_schedule.marginal_alpha(t_continuous), noise_schedule.marginal_std(t_continuous)
+            dims = x.dim()
+            return expand_dims(alpha_t, dims) * output + expand_dims(sigma_t, dims) * x
+        elif model_type == "score":
+            sigma_t = noise_schedule.marginal_std(t_continuous)
+            dims = x.dim()
+            return -expand_dims(sigma_t, dims) * output
+
+    def cond_grad_fn(x, t_input):
+        """
+        Compute the gradient of the classifier, i.e. nabla_{x} log p_t(cond | x_t).
+        """
+        with torch.enable_grad():
+            x_in = x.detach().requires_grad_(True)
+            log_prob = classifier_fn(x_in, t_input, condition, **classifier_kwargs)
+            return torch.autograd.grad(log_prob.sum(), x_in)[0]
+
+    def model_fn(x, t_continuous):
+        """
+        The noise predicition model function that is used for DPM-Solver.
+        """
+        if t_continuous.reshape((-1,)).shape[0] == 1:
+            t_continuous = t_continuous.expand((x.shape[0]))
+        if guidance_type == "uncond":
+            return noise_pred_fn(x, t_continuous)
+        elif guidance_type == "classifier":
+            assert classifier_fn is not None
+            t_input = get_model_input_time(t_continuous)
+            cond_grad = cond_grad_fn(x, t_input)
+            sigma_t = noise_schedule.marginal_std(t_continuous)
+            noise = noise_pred_fn(x, t_continuous)
+            return noise - guidance_scale * expand_dims(sigma_t, dims=cond_grad.dim()) * cond_grad
+        elif guidance_type == "classifier-free":
+            if guidance_scale == 1. or unconditional_condition is None:
+                return noise_pred_fn(x, t_continuous, cond=condition)
+            else:
+                x_in = torch.cat([x] * 2)
+                t_in = torch.cat([t_continuous] * 2)
+                c_in = torch.cat([unconditional_condition, condition])
+                noise_uncond, noise = noise_pred_fn(x_in, t_in, cond=c_in).chunk(2)
+                return noise_uncond + guidance_scale * (noise - noise_uncond)
+
+    assert model_type in ["noise", "x_start", "v"]
+    assert guidance_type in ["uncond", "classifier", "classifier-free"]
+    return model_fn
+
+
+class DPM_Solver:
+    def __init__(self, model_fn, noise_schedule, predict_x0=False, thresholding=False, max_val=1.):
+        """Construct a DPM-Solver. 
+
+        We support both the noise prediction model ("predicting epsilon") and the data prediction model ("predicting x0").
+        If `predict_x0` is False, we use the solver for the noise prediction model (DPM-Solver).
+        If `predict_x0` is True, we use the solver for the data prediction model (DPM-Solver++).
+            In such case, we further support the "dynamic thresholding" in [1] when `thresholding` is True.
+            The "dynamic thresholding" can greatly improve the sample quality for pixel-space DPMs with large guidance scales.
+
+        Args:
+            model_fn: A noise prediction model function which accepts the continuous-time input (t in [epsilon, T]):
+                ``
+                def model_fn(x, t_continuous):
+                    return noise
+                ``
+            noise_schedule: A noise schedule object, such as NoiseScheduleVP.
+            predict_x0: A `bool`. If true, use the data prediction model; else, use the noise prediction model.
+            thresholding: A `bool`. Valid when `predict_x0` is True. Whether to use the "dynamic thresholding" in [1].
+            max_val: A `float`. Valid when both `predict_x0` and `thresholding` are True. The max value for thresholding.
+        
+        [1] Chitwan Saharia, William Chan, Saurabh Saxena, Lala Li, Jay Whang, Emily Denton, Seyed Kamyar Seyed Ghasemipour, Burcu Karagol Ayan, S Sara Mahdavi, Rapha Gontijo Lopes, et al. Photorealistic text-to-image diffusion models with deep language understanding. arXiv preprint arXiv:2205.11487, 2022b.
+        """
+        self.model = model_fn
+        self.noise_schedule = noise_schedule
+        self.predict_x0 = predict_x0
+        self.thresholding = thresholding
+        self.max_val = max_val
+
+    def noise_prediction_fn(self, x, t):
+        """
+        Return the noise prediction model.
+        """
+        return self.model(x, t)
+
+    def data_prediction_fn(self, x, t):
+        """
+        Return the data prediction model (with thresholding).
+        """
+        noise = self.noise_prediction_fn(x, t)
+        dims = x.dim()
+        alpha_t, sigma_t = self.noise_schedule.marginal_alpha(t), self.noise_schedule.marginal_std(t)
+        x0 = (x - expand_dims(sigma_t, dims) * noise) / expand_dims(alpha_t, dims)
+        if self.thresholding:
+            p = 0.995  # A hyperparameter in the paper of "Imagen" [1].
+            s = torch.quantile(torch.abs(x0).reshape((x0.shape[0], -1)), p, dim=1)
+            s = expand_dims(torch.maximum(s, self.max_val * torch.ones_like(s).to(s.device)), dims)
+            x0 = torch.clamp(x0, -s, s) / s
+        return x0
+
+    def model_fn(self, x, t):
+        """
+        Convert the model to the noise prediction model or the data prediction model. 
+        """
+        if self.predict_x0:
+            return self.data_prediction_fn(x, t)
+        else:
+            return self.noise_prediction_fn(x, t)
+
+    def get_time_steps(self, skip_type, t_T, t_0, N, device):
+        """Compute the intermediate time steps for sampling.
+
+        Args:
+            skip_type: A `str`. The type for the spacing of the time steps. We support three types:
+                - 'logSNR': uniform logSNR for the time steps.
+                - 'time_uniform': uniform time for the time steps. (**Recommended for high-resolutional data**.)
+                - 'time_quadratic': quadratic time for the time steps. (Used in DDIM for low-resolutional data.)
+            t_T: A `float`. The starting time of the sampling (default is T).
+            t_0: A `float`. The ending time of the sampling (default is epsilon).
+            N: A `int`. The total number of the spacing of the time steps.
+            device: A torch device.
+        Returns:
+            A pytorch tensor of the time steps, with the shape (N + 1,).
+        """
+        if skip_type == 'logSNR':
+            lambda_T = self.noise_schedule.marginal_lambda(torch.tensor(t_T).to(device))
+            lambda_0 = self.noise_schedule.marginal_lambda(torch.tensor(t_0).to(device))
+            logSNR_steps = torch.linspace(lambda_T.cpu().item(), lambda_0.cpu().item(), N + 1).to(device)
+            return self.noise_schedule.inverse_lambda(logSNR_steps)
+        elif skip_type == 'time_uniform':
+            return torch.linspace(t_T, t_0, N + 1).to(device)
+        elif skip_type == 'time_quadratic':
+            t_order = 2
+            t = torch.linspace(t_T ** (1. / t_order), t_0 ** (1. / t_order), N + 1).pow(t_order).to(device)
+            return t
+        else:
+            raise ValueError(
+                "Unsupported skip_type {}, need to be 'logSNR' or 'time_uniform' or 'time_quadratic'".format(skip_type))
+
+    def get_orders_and_timesteps_for_singlestep_solver(self, steps, order, skip_type, t_T, t_0, device):
+        """
+        Get the order of each step for sampling by the singlestep DPM-Solver.
+
+        We combine both DPM-Solver-1,2,3 to use all the function evaluations, which is named as "DPM-Solver-fast".
+        Given a fixed number of function evaluations by `steps`, the sampling procedure by DPM-Solver-fast is:
+            - If order == 1:
+                We take `steps` of DPM-Solver-1 (i.e. DDIM).
+            - If order == 2:
+                - Denote K = (steps // 2). We take K or (K + 1) intermediate time steps for sampling.
+                - If steps % 2 == 0, we use K steps of DPM-Solver-2.
+                - If steps % 2 == 1, we use K steps of DPM-Solver-2 and 1 step of DPM-Solver-1.
+            - If order == 3:
+                - Denote K = (steps // 3 + 1). We take K intermediate time steps for sampling.
+                - If steps % 3 == 0, we use (K - 2) steps of DPM-Solver-3, and 1 step of DPM-Solver-2 and 1 step of DPM-Solver-1.
+                - If steps % 3 == 1, we use (K - 1) steps of DPM-Solver-3 and 1 step of DPM-Solver-1.
+                - If steps % 3 == 2, we use (K - 1) steps of DPM-Solver-3 and 1 step of DPM-Solver-2.
+
+        ============================================
+        Args:
+            order: A `int`. The max order for the solver (2 or 3).
+            steps: A `int`. The total number of function evaluations (NFE).
+            skip_type: A `str`. The type for the spacing of the time steps. We support three types:
+                - 'logSNR': uniform logSNR for the time steps.
+                - 'time_uniform': uniform time for the time steps. (**Recommended for high-resolutional data**.)
+                - 'time_quadratic': quadratic time for the time steps. (Used in DDIM for low-resolutional data.)
+            t_T: A `float`. The starting time of the sampling (default is T).
+            t_0: A `float`. The ending time of the sampling (default is epsilon).
+            device: A torch device.
+        Returns:
+            orders: A list of the solver order of each step.
+        """
+        if order == 3:
+            K = steps // 3 + 1
+            if steps % 3 == 0:
+                orders = [3, ] * (K - 2) + [2, 1]
+            elif steps % 3 == 1:
+                orders = [3, ] * (K - 1) + [1]
+            else:
+                orders = [3, ] * (K - 1) + [2]
+        elif order == 2:
+            if steps % 2 == 0:
+                K = steps // 2
+                orders = [2, ] * K
+            else:
+                K = steps // 2 + 1
+                orders = [2, ] * (K - 1) + [1]
+        elif order == 1:
+            K = 1
+            orders = [1, ] * steps
+        else:
+            raise ValueError("'order' must be '1' or '2' or '3'.")
+        if skip_type == 'logSNR':
+            # To reproduce the results in DPM-Solver paper
+            timesteps_outer = self.get_time_steps(skip_type, t_T, t_0, K, device)
+        else:
+            timesteps_outer = self.get_time_steps(skip_type, t_T, t_0, steps, device)[
+                torch.cumsum(torch.tensor([0, ] + orders), dim=0).to(device)]
+        return timesteps_outer, orders
+
+    def denoise_fn(self, x, s):
+        """
+        Denoise at the final step, which is equivalent to solve the ODE from lambda_s to infty by first-order discretization. 
+        """
+        return self.data_prediction_fn(x, s)
+
+    def dpm_solver_first_update(self, x, s, t, model_s=None, return_intermediate=False):
+        """
+        DPM-Solver-1 (equivalent to DDIM) from time `s` to time `t`.
+
+        Args:
+            x: A pytorch tensor. The initial value at time `s`.
+            s: A pytorch tensor. The starting time, with the shape (x.shape[0],).
+            t: A pytorch tensor. The ending time, with the shape (x.shape[0],).
+            model_s: A pytorch tensor. The model function evaluated at time `s`.
+                If `model_s` is None, we evaluate the model by `x` and `s`; otherwise we directly use it.
+            return_intermediate: A `bool`. If true, also return the model value at time `s`.
+        Returns:
+            x_t: A pytorch tensor. The approximated solution at time `t`.
+        """
+        ns = self.noise_schedule
+        dims = x.dim()
+        lambda_s, lambda_t = ns.marginal_lambda(s), ns.marginal_lambda(t)
+        h = lambda_t - lambda_s
+        log_alpha_s, log_alpha_t = ns.marginal_log_mean_coeff(s), ns.marginal_log_mean_coeff(t)
+        sigma_s, sigma_t = ns.marginal_std(s), ns.marginal_std(t)
+        alpha_t = torch.exp(log_alpha_t)
+
+        if self.predict_x0:
+            phi_1 = torch.expm1(-h)
+            if model_s is None:
+                model_s = self.model_fn(x, s)
+            x_t = (
+                    expand_dims(sigma_t / sigma_s, dims) * x
+                    - expand_dims(alpha_t * phi_1, dims) * model_s
+            )
+            if return_intermediate:
+                return x_t, {'model_s': model_s}
+            else:
+                return x_t
+        else:
+            phi_1 = torch.expm1(h)
+            if model_s is None:
+                model_s = self.model_fn(x, s)
+            x_t = (
+                    expand_dims(torch.exp(log_alpha_t - log_alpha_s), dims) * x
+                    - expand_dims(sigma_t * phi_1, dims) * model_s
+            )
+            if return_intermediate:
+                return x_t, {'model_s': model_s}
+            else:
+                return x_t
+
+    def singlestep_dpm_solver_second_update(self, x, s, t, r1=0.5, model_s=None, return_intermediate=False,
+                                            solver_type='dpm_solver'):
+        """
+        Singlestep solver DPM-Solver-2 from time `s` to time `t`.
+
+        Args:
+            x: A pytorch tensor. The initial value at time `s`.
+            s: A pytorch tensor. The starting time, with the shape (x.shape[0],).
+            t: A pytorch tensor. The ending time, with the shape (x.shape[0],).
+            r1: A `float`. The hyperparameter of the second-order solver.
+            model_s: A pytorch tensor. The model function evaluated at time `s`.
+                If `model_s` is None, we evaluate the model by `x` and `s`; otherwise we directly use it.
+            return_intermediate: A `bool`. If true, also return the model value at time `s` and `s1` (the intermediate time).
+            solver_type: either 'dpm_solver' or 'taylor'. The type for the high-order solvers.
+                The type slightly impacts the performance. We recommend to use 'dpm_solver' type.
+        Returns:
+            x_t: A pytorch tensor. The approximated solution at time `t`.
+        """
+        if solver_type not in ['dpm_solver', 'taylor']:
+            raise ValueError("'solver_type' must be either 'dpm_solver' or 'taylor', got {}".format(solver_type))
+        if r1 is None:
+            r1 = 0.5
+        ns = self.noise_schedule
+        dims = x.dim()
+        lambda_s, lambda_t = ns.marginal_lambda(s), ns.marginal_lambda(t)
+        h = lambda_t - lambda_s
+        lambda_s1 = lambda_s + r1 * h
+        s1 = ns.inverse_lambda(lambda_s1)
+        log_alpha_s, log_alpha_s1, log_alpha_t = ns.marginal_log_mean_coeff(s), ns.marginal_log_mean_coeff(
+            s1), ns.marginal_log_mean_coeff(t)
+        sigma_s, sigma_s1, sigma_t = ns.marginal_std(s), ns.marginal_std(s1), ns.marginal_std(t)
+        alpha_s1, alpha_t = torch.exp(log_alpha_s1), torch.exp(log_alpha_t)
+
+        if self.predict_x0:
+            phi_11 = torch.expm1(-r1 * h)
+            phi_1 = torch.expm1(-h)
+
+            if model_s is None:
+                model_s = self.model_fn(x, s)
+            x_s1 = (
+                    expand_dims(sigma_s1 / sigma_s, dims) * x
+                    - expand_dims(alpha_s1 * phi_11, dims) * model_s
+            )
+            model_s1 = self.model_fn(x_s1, s1)
+            if solver_type == 'dpm_solver':
+                x_t = (
+                        expand_dims(sigma_t / sigma_s, dims) * x
+                        - expand_dims(alpha_t * phi_1, dims) * model_s
+                        - (0.5 / r1) * expand_dims(alpha_t * phi_1, dims) * (model_s1 - model_s)
+                )
+            elif solver_type == 'taylor':
+                x_t = (
+                        expand_dims(sigma_t / sigma_s, dims) * x
+                        - expand_dims(alpha_t * phi_1, dims) * model_s
+                        + (1. / r1) * expand_dims(alpha_t * ((torch.exp(-h) - 1.) / h + 1.), dims) * (
+                                    model_s1 - model_s)
+                )
+        else:
+            phi_11 = torch.expm1(r1 * h)
+            phi_1 = torch.expm1(h)
+
+            if model_s is None:
+                model_s = self.model_fn(x, s)
+            x_s1 = (
+                    expand_dims(torch.exp(log_alpha_s1 - log_alpha_s), dims) * x
+                    - expand_dims(sigma_s1 * phi_11, dims) * model_s
+            )
+            model_s1 = self.model_fn(x_s1, s1)
+            if solver_type == 'dpm_solver':
+                x_t = (
+                        expand_dims(torch.exp(log_alpha_t - log_alpha_s), dims) * x
+                        - expand_dims(sigma_t * phi_1, dims) * model_s
+                        - (0.5 / r1) * expand_dims(sigma_t * phi_1, dims) * (model_s1 - model_s)
+                )
+            elif solver_type == 'taylor':
+                x_t = (
+                        expand_dims(torch.exp(log_alpha_t - log_alpha_s), dims) * x
+                        - expand_dims(sigma_t * phi_1, dims) * model_s
+                        - (1. / r1) * expand_dims(sigma_t * ((torch.exp(h) - 1.) / h - 1.), dims) * (model_s1 - model_s)
+                )
+        if return_intermediate:
+            return x_t, {'model_s': model_s, 'model_s1': model_s1}
+        else:
+            return x_t
+
+    def singlestep_dpm_solver_third_update(self, x, s, t, r1=1. / 3., r2=2. / 3., model_s=None, model_s1=None,
+                                           return_intermediate=False, solver_type='dpm_solver'):
+        """
+        Singlestep solver DPM-Solver-3 from time `s` to time `t`.
+
+        Args:
+            x: A pytorch tensor. The initial value at time `s`.
+            s: A pytorch tensor. The starting time, with the shape (x.shape[0],).
+            t: A pytorch tensor. The ending time, with the shape (x.shape[0],).
+            r1: A `float`. The hyperparameter of the third-order solver.
+            r2: A `float`. The hyperparameter of the third-order solver.
+            model_s: A pytorch tensor. The model function evaluated at time `s`.
+                If `model_s` is None, we evaluate the model by `x` and `s`; otherwise we directly use it.
+            model_s1: A pytorch tensor. The model function evaluated at time `s1` (the intermediate time given by `r1`).
+                If `model_s1` is None, we evaluate the model at `s1`; otherwise we directly use it.
+            return_intermediate: A `bool`. If true, also return the model value at time `s`, `s1` and `s2` (the intermediate times).
+            solver_type: either 'dpm_solver' or 'taylor'. The type for the high-order solvers.
+                The type slightly impacts the performance. We recommend to use 'dpm_solver' type.
+        Returns:
+            x_t: A pytorch tensor. The approximated solution at time `t`.
+        """
+        if solver_type not in ['dpm_solver', 'taylor']:
+            raise ValueError("'solver_type' must be either 'dpm_solver' or 'taylor', got {}".format(solver_type))
+        if r1 is None:
+            r1 = 1. / 3.
+        if r2 is None:
+            r2 = 2. / 3.
+        ns = self.noise_schedule
+        dims = x.dim()
+        lambda_s, lambda_t = ns.marginal_lambda(s), ns.marginal_lambda(t)
+        h = lambda_t - lambda_s
+        lambda_s1 = lambda_s + r1 * h
+        lambda_s2 = lambda_s + r2 * h
+        s1 = ns.inverse_lambda(lambda_s1)
+        s2 = ns.inverse_lambda(lambda_s2)
+        log_alpha_s, log_alpha_s1, log_alpha_s2, log_alpha_t = ns.marginal_log_mean_coeff(
+            s), ns.marginal_log_mean_coeff(s1), ns.marginal_log_mean_coeff(s2), ns.marginal_log_mean_coeff(t)
+        sigma_s, sigma_s1, sigma_s2, sigma_t = ns.marginal_std(s), ns.marginal_std(s1), ns.marginal_std(
+            s2), ns.marginal_std(t)
+        alpha_s1, alpha_s2, alpha_t = torch.exp(log_alpha_s1), torch.exp(log_alpha_s2), torch.exp(log_alpha_t)
+
+        if self.predict_x0:
+            phi_11 = torch.expm1(-r1 * h)
+            phi_12 = torch.expm1(-r2 * h)
+            phi_1 = torch.expm1(-h)
+            phi_22 = torch.expm1(-r2 * h) / (r2 * h) + 1.
+            phi_2 = phi_1 / h + 1.
+            phi_3 = phi_2 / h - 0.5
+
+            if model_s is None:
+                model_s = self.model_fn(x, s)
+            if model_s1 is None:
+                x_s1 = (
+                        expand_dims(sigma_s1 / sigma_s, dims) * x
+                        - expand_dims(alpha_s1 * phi_11, dims) * model_s
+                )
+                model_s1 = self.model_fn(x_s1, s1)
+            x_s2 = (
+                    expand_dims(sigma_s2 / sigma_s, dims) * x
+                    - expand_dims(alpha_s2 * phi_12, dims) * model_s
+                    + r2 / r1 * expand_dims(alpha_s2 * phi_22, dims) * (model_s1 - model_s)
+            )
+            model_s2 = self.model_fn(x_s2, s2)
+            if solver_type == 'dpm_solver':
+                x_t = (
+                        expand_dims(sigma_t / sigma_s, dims) * x
+                        - expand_dims(alpha_t * phi_1, dims) * model_s
+                        + (1. / r2) * expand_dims(alpha_t * phi_2, dims) * (model_s2 - model_s)
+                )
+            elif solver_type == 'taylor':
+                D1_0 = (1. / r1) * (model_s1 - model_s)
+                D1_1 = (1. / r2) * (model_s2 - model_s)
+                D1 = (r2 * D1_0 - r1 * D1_1) / (r2 - r1)
+                D2 = 2. * (D1_1 - D1_0) / (r2 - r1)
+                x_t = (
+                        expand_dims(sigma_t / sigma_s, dims) * x
+                        - expand_dims(alpha_t * phi_1, dims) * model_s
+                        + expand_dims(alpha_t * phi_2, dims) * D1
+                        - expand_dims(alpha_t * phi_3, dims) * D2
+                )
+        else:
+            phi_11 = torch.expm1(r1 * h)
+            phi_12 = torch.expm1(r2 * h)
+            phi_1 = torch.expm1(h)
+            phi_22 = torch.expm1(r2 * h) / (r2 * h) - 1.
+            phi_2 = phi_1 / h - 1.
+            phi_3 = phi_2 / h - 0.5
+
+            if model_s is None:
+                model_s = self.model_fn(x, s)
+            if model_s1 is None:
+                x_s1 = (
+                        expand_dims(torch.exp(log_alpha_s1 - log_alpha_s), dims) * x
+                        - expand_dims(sigma_s1 * phi_11, dims) * model_s
+                )
+                model_s1 = self.model_fn(x_s1, s1)
+            x_s2 = (
+                    expand_dims(torch.exp(log_alpha_s2 - log_alpha_s), dims) * x
+                    - expand_dims(sigma_s2 * phi_12, dims) * model_s
+                    - r2 / r1 * expand_dims(sigma_s2 * phi_22, dims) * (model_s1 - model_s)
+            )
+            model_s2 = self.model_fn(x_s2, s2)
+            if solver_type == 'dpm_solver':
+                x_t = (
+                        expand_dims(torch.exp(log_alpha_t - log_alpha_s), dims) * x
+                        - expand_dims(sigma_t * phi_1, dims) * model_s
+                        - (1. / r2) * expand_dims(sigma_t * phi_2, dims) * (model_s2 - model_s)
+                )
+            elif solver_type == 'taylor':
+                D1_0 = (1. / r1) * (model_s1 - model_s)
+                D1_1 = (1. / r2) * (model_s2 - model_s)
+                D1 = (r2 * D1_0 - r1 * D1_1) / (r2 - r1)
+                D2 = 2. * (D1_1 - D1_0) / (r2 - r1)
+                x_t = (
+                        expand_dims(torch.exp(log_alpha_t - log_alpha_s), dims) * x
+                        - expand_dims(sigma_t * phi_1, dims) * model_s
+                        - expand_dims(sigma_t * phi_2, dims) * D1
+                        - expand_dims(sigma_t * phi_3, dims) * D2
+                )
+
+        if return_intermediate:
+            return x_t, {'model_s': model_s, 'model_s1': model_s1, 'model_s2': model_s2}
+        else:
+            return x_t
+
+    def multistep_dpm_solver_second_update(self, x, model_prev_list, t_prev_list, t, solver_type="dpm_solver"):
+        """
+        Multistep solver DPM-Solver-2 from time `t_prev_list[-1]` to time `t`.
+
+        Args:
+            x: A pytorch tensor. The initial value at time `s`.
+            model_prev_list: A list of pytorch tensor. The previous computed model values.
+            t_prev_list: A list of pytorch tensor. The previous times, each time has the shape (x.shape[0],)
+            t: A pytorch tensor. The ending time, with the shape (x.shape[0],).
+            solver_type: either 'dpm_solver' or 'taylor'. The type for the high-order solvers.
+                The type slightly impacts the performance. We recommend to use 'dpm_solver' type.
+        Returns:
+            x_t: A pytorch tensor. The approximated solution at time `t`.
+        """
+        if solver_type not in ['dpm_solver', 'taylor']:
+            raise ValueError("'solver_type' must be either 'dpm_solver' or 'taylor', got {}".format(solver_type))
+        ns = self.noise_schedule
+        dims = x.dim()
+        model_prev_1, model_prev_0 = model_prev_list
+        t_prev_1, t_prev_0 = t_prev_list
+        lambda_prev_1, lambda_prev_0, lambda_t = ns.marginal_lambda(t_prev_1), ns.marginal_lambda(
+            t_prev_0), ns.marginal_lambda(t)
+        log_alpha_prev_0, log_alpha_t = ns.marginal_log_mean_coeff(t_prev_0), ns.marginal_log_mean_coeff(t)
+        sigma_prev_0, sigma_t = ns.marginal_std(t_prev_0), ns.marginal_std(t)
+        alpha_t = torch.exp(log_alpha_t)
+
+        h_0 = lambda_prev_0 - lambda_prev_1
+        h = lambda_t - lambda_prev_0
+        r0 = h_0 / h
+        D1_0 = expand_dims(1. / r0, dims) * (model_prev_0 - model_prev_1)
+        if self.predict_x0:
+            if solver_type == 'dpm_solver':
+                x_t = (
+                        expand_dims(sigma_t / sigma_prev_0, dims) * x
+                        - expand_dims(alpha_t * (torch.exp(-h) - 1.), dims) * model_prev_0
+                        - 0.5 * expand_dims(alpha_t * (torch.exp(-h) - 1.), dims) * D1_0
+                )
+            elif solver_type == 'taylor':
+                x_t = (
+                        expand_dims(sigma_t / sigma_prev_0, dims) * x
+                        - expand_dims(alpha_t * (torch.exp(-h) - 1.), dims) * model_prev_0
+                        + expand_dims(alpha_t * ((torch.exp(-h) - 1.) / h + 1.), dims) * D1_0
+                )
+        else:
+            if solver_type == 'dpm_solver':
+                x_t = (
+                        expand_dims(torch.exp(log_alpha_t - log_alpha_prev_0), dims) * x
+                        - expand_dims(sigma_t * (torch.exp(h) - 1.), dims) * model_prev_0
+                        - 0.5 * expand_dims(sigma_t * (torch.exp(h) - 1.), dims) * D1_0
+                )
+            elif solver_type == 'taylor':
+                x_t = (
+                        expand_dims(torch.exp(log_alpha_t - log_alpha_prev_0), dims) * x
+                        - expand_dims(sigma_t * (torch.exp(h) - 1.), dims) * model_prev_0
+                        - expand_dims(sigma_t * ((torch.exp(h) - 1.) / h - 1.), dims) * D1_0
+                )
+        return x_t
+
+    def multistep_dpm_solver_third_update(self, x, model_prev_list, t_prev_list, t, solver_type='dpm_solver'):
+        """
+        Multistep solver DPM-Solver-3 from time `t_prev_list[-1]` to time `t`.
+
+        Args:
+            x: A pytorch tensor. The initial value at time `s`.
+            model_prev_list: A list of pytorch tensor. The previous computed model values.
+            t_prev_list: A list of pytorch tensor. The previous times, each time has the shape (x.shape[0],)
+            t: A pytorch tensor. The ending time, with the shape (x.shape[0],).
+            solver_type: either 'dpm_solver' or 'taylor'. The type for the high-order solvers.
+                The type slightly impacts the performance. We recommend to use 'dpm_solver' type.
+        Returns:
+            x_t: A pytorch tensor. The approximated solution at time `t`.
+        """
+        ns = self.noise_schedule
+        dims = x.dim()
+        model_prev_2, model_prev_1, model_prev_0 = model_prev_list
+        t_prev_2, t_prev_1, t_prev_0 = t_prev_list
+        lambda_prev_2, lambda_prev_1, lambda_prev_0, lambda_t = ns.marginal_lambda(t_prev_2), ns.marginal_lambda(
+            t_prev_1), ns.marginal_lambda(t_prev_0), ns.marginal_lambda(t)
+        log_alpha_prev_0, log_alpha_t = ns.marginal_log_mean_coeff(t_prev_0), ns.marginal_log_mean_coeff(t)
+        sigma_prev_0, sigma_t = ns.marginal_std(t_prev_0), ns.marginal_std(t)
+        alpha_t = torch.exp(log_alpha_t)
+
+        h_1 = lambda_prev_1 - lambda_prev_2
+        h_0 = lambda_prev_0 - lambda_prev_1
+        h = lambda_t - lambda_prev_0
+        r0, r1 = h_0 / h, h_1 / h
+        D1_0 = expand_dims(1. / r0, dims) * (model_prev_0 - model_prev_1)
+        D1_1 = expand_dims(1. / r1, dims) * (model_prev_1 - model_prev_2)
+        D1 = D1_0 + expand_dims(r0 / (r0 + r1), dims) * (D1_0 - D1_1)
+        D2 = expand_dims(1. / (r0 + r1), dims) * (D1_0 - D1_1)
+        if self.predict_x0:
+            x_t = (
+                    expand_dims(sigma_t / sigma_prev_0, dims) * x
+                    - expand_dims(alpha_t * (torch.exp(-h) - 1.), dims) * model_prev_0
+                    + expand_dims(alpha_t * ((torch.exp(-h) - 1.) / h + 1.), dims) * D1
+                    - expand_dims(alpha_t * ((torch.exp(-h) - 1. + h) / h ** 2 - 0.5), dims) * D2
+            )
+        else:
+            x_t = (
+                    expand_dims(torch.exp(log_alpha_t - log_alpha_prev_0), dims) * x
+                    - expand_dims(sigma_t * (torch.exp(h) - 1.), dims) * model_prev_0
+                    - expand_dims(sigma_t * ((torch.exp(h) - 1.) / h - 1.), dims) * D1
+                    - expand_dims(sigma_t * ((torch.exp(h) - 1. - h) / h ** 2 - 0.5), dims) * D2
+            )
+        return x_t
+
+    def singlestep_dpm_solver_update(self, x, s, t, order, return_intermediate=False, solver_type='dpm_solver', r1=None,
+                                     r2=None):
+        """
+        Singlestep DPM-Solver with the order `order` from time `s` to time `t`.
+
+        Args:
+            x: A pytorch tensor. The initial value at time `s`.
+            s: A pytorch tensor. The starting time, with the shape (x.shape[0],).
+            t: A pytorch tensor. The ending time, with the shape (x.shape[0],).
+            order: A `int`. The order of DPM-Solver. We only support order == 1 or 2 or 3.
+            return_intermediate: A `bool`. If true, also return the model value at time `s`, `s1` and `s2` (the intermediate times).
+            solver_type: either 'dpm_solver' or 'taylor'. The type for the high-order solvers.
+                The type slightly impacts the performance. We recommend to use 'dpm_solver' type.
+            r1: A `float`. The hyperparameter of the second-order or third-order solver.
+            r2: A `float`. The hyperparameter of the third-order solver.
+        Returns:
+            x_t: A pytorch tensor. The approximated solution at time `t`.
+        """
+        if order == 1:
+            return self.dpm_solver_first_update(x, s, t, return_intermediate=return_intermediate)
+        elif order == 2:
+            return self.singlestep_dpm_solver_second_update(x, s, t, return_intermediate=return_intermediate,
+                                                            solver_type=solver_type, r1=r1)
+        elif order == 3:
+            return self.singlestep_dpm_solver_third_update(x, s, t, return_intermediate=return_intermediate,
+                                                           solver_type=solver_type, r1=r1, r2=r2)
+        else:
+            raise ValueError("Solver order must be 1 or 2 or 3, got {}".format(order))
+
+    def multistep_dpm_solver_update(self, x, model_prev_list, t_prev_list, t, order, solver_type='dpm_solver'):
+        """
+        Multistep DPM-Solver with the order `order` from time `t_prev_list[-1]` to time `t`.
+
+        Args:
+            x: A pytorch tensor. The initial value at time `s`.
+            model_prev_list: A list of pytorch tensor. The previous computed model values.
+            t_prev_list: A list of pytorch tensor. The previous times, each time has the shape (x.shape[0],)
+            t: A pytorch tensor. The ending time, with the shape (x.shape[0],).
+            order: A `int`. The order of DPM-Solver. We only support order == 1 or 2 or 3.
+            solver_type: either 'dpm_solver' or 'taylor'. The type for the high-order solvers.
+                The type slightly impacts the performance. We recommend to use 'dpm_solver' type.
+        Returns:
+            x_t: A pytorch tensor. The approximated solution at time `t`.
+        """
+        if order == 1:
+            return self.dpm_solver_first_update(x, t_prev_list[-1], t, model_s=model_prev_list[-1])
+        elif order == 2:
+            return self.multistep_dpm_solver_second_update(x, model_prev_list, t_prev_list, t, solver_type=solver_type)
+        elif order == 3:
+            return self.multistep_dpm_solver_third_update(x, model_prev_list, t_prev_list, t, solver_type=solver_type)
+        else:
+            raise ValueError("Solver order must be 1 or 2 or 3, got {}".format(order))
+
+    def dpm_solver_adaptive(self, x, order, t_T, t_0, h_init=0.05, atol=0.0078, rtol=0.05, theta=0.9, t_err=1e-5,
+                            solver_type='dpm_solver'):
+        """
+        The adaptive step size solver based on singlestep DPM-Solver.
+
+        Args:
+            x: A pytorch tensor. The initial value at time `t_T`.
+            order: A `int`. The (higher) order of the solver. We only support order == 2 or 3.
+            t_T: A `float`. The starting time of the sampling (default is T).
+            t_0: A `float`. The ending time of the sampling (default is epsilon).
+            h_init: A `float`. The initial step size (for logSNR).
+            atol: A `float`. The absolute tolerance of the solver. For image data, the default setting is 0.0078, followed [1].
+            rtol: A `float`. The relative tolerance of the solver. The default setting is 0.05.
+            theta: A `float`. The safety hyperparameter for adapting the step size. The default setting is 0.9, followed [1].
+            t_err: A `float`. The tolerance for the time. We solve the diffusion ODE until the absolute error between the 
+                current time and `t_0` is less than `t_err`. The default setting is 1e-5.
+            solver_type: either 'dpm_solver' or 'taylor'. The type for the high-order solvers.
+                The type slightly impacts the performance. We recommend to use 'dpm_solver' type.
+        Returns:
+            x_0: A pytorch tensor. The approximated solution at time `t_0`.
+
+        [1] A. Jolicoeur-Martineau, K. Li, R. Piché-Taillefer, T. Kachman, and I. Mitliagkas, "Gotta go fast when generating data with score-based models," arXiv preprint arXiv:2105.14080, 2021.
+        """
+        ns = self.noise_schedule
+        s = t_T * torch.ones((x.shape[0],)).to(x)
+        lambda_s = ns.marginal_lambda(s)
+        lambda_0 = ns.marginal_lambda(t_0 * torch.ones_like(s).to(x))
+        h = h_init * torch.ones_like(s).to(x)
+        x_prev = x
+        nfe = 0
+        if order == 2:
+            r1 = 0.5
+            lower_update = lambda x, s, t: self.dpm_solver_first_update(x, s, t, return_intermediate=True)
+            higher_update = lambda x, s, t, **kwargs: self.singlestep_dpm_solver_second_update(x, s, t, r1=r1,
+                                                                                               solver_type=solver_type,
+                                                                                               **kwargs)
+        elif order == 3:
+            r1, r2 = 1. / 3., 2. / 3.
+            lower_update = lambda x, s, t: self.singlestep_dpm_solver_second_update(x, s, t, r1=r1,
+                                                                                    return_intermediate=True,
+                                                                                    solver_type=solver_type)
+            higher_update = lambda x, s, t, **kwargs: self.singlestep_dpm_solver_third_update(x, s, t, r1=r1, r2=r2,
+                                                                                              solver_type=solver_type,
+                                                                                              **kwargs)
+        else:
+            raise ValueError("For adaptive step size solver, order must be 2 or 3, got {}".format(order))
+        while torch.abs((s - t_0)).mean() > t_err:
+            t = ns.inverse_lambda(lambda_s + h)
+            x_lower, lower_noise_kwargs = lower_update(x, s, t)
+            x_higher = higher_update(x, s, t, **lower_noise_kwargs)
+            delta = torch.max(torch.ones_like(x).to(x) * atol, rtol * torch.max(torch.abs(x_lower), torch.abs(x_prev)))
+            norm_fn = lambda v: torch.sqrt(torch.square(v.reshape((v.shape[0], -1))).mean(dim=-1, keepdim=True))
+            E = norm_fn((x_higher - x_lower) / delta).max()
+            if torch.all(E <= 1.):
+                x = x_higher
+                s = t
+                x_prev = x_lower
+                lambda_s = ns.marginal_lambda(s)
+            h = torch.min(theta * h * torch.float_power(E, -1. / order).float(), lambda_0 - lambda_s)
+            nfe += order
+        print('adaptive solver nfe', nfe)
+        return x
+
+    def sample(self, x, steps=20, t_start=None, t_end=None, order=3, skip_type='time_uniform',
+               method='singlestep', denoise=False, solver_type='dpm_solver', atol=0.0078,
+               rtol=0.05,
+               ):
+        """
+        Compute the sample at time `t_end` by DPM-Solver, given the initial `x` at time `t_start`.
+
+        =====================================================
+
+        We support the following algorithms for both noise prediction model and data prediction model:
+            - 'singlestep':
+                Singlestep DPM-Solver (i.e. "DPM-Solver-fast" in the paper), which combines different orders of singlestep DPM-Solver. 
+                We combine all the singlestep solvers with order <= `order` to use up all the function evaluations (steps).
+                The total number of function evaluations (NFE) == `steps`.
+                Given a fixed NFE == `steps`, the sampling procedure is:
+                    - If `order` == 1:
+                        - Denote K = steps. We use K steps of DPM-Solver-1 (i.e. DDIM).
+                    - If `order` == 2:
+                        - Denote K = (steps // 2) + (steps % 2). We take K intermediate time steps for sampling.
+                        - If steps % 2 == 0, we use K steps of singlestep DPM-Solver-2.
+                        - If steps % 2 == 1, we use (K - 1) steps of singlestep DPM-Solver-2 and 1 step of DPM-Solver-1.
+                    - If `order` == 3:
+                        - Denote K = (steps // 3 + 1). We take K intermediate time steps for sampling.
+                        - If steps % 3 == 0, we use (K - 2) steps of singlestep DPM-Solver-3, and 1 step of singlestep DPM-Solver-2 and 1 step of DPM-Solver-1.
+                        - If steps % 3 == 1, we use (K - 1) steps of singlestep DPM-Solver-3 and 1 step of DPM-Solver-1.
+                        - If steps % 3 == 2, we use (K - 1) steps of singlestep DPM-Solver-3 and 1 step of singlestep DPM-Solver-2.
+            - 'multistep':
+                Multistep DPM-Solver with the order of `order`. The total number of function evaluations (NFE) == `steps`.
+                We initialize the first `order` values by lower order multistep solvers.
+                Given a fixed NFE == `steps`, the sampling procedure is:
+                    Denote K = steps.
+                    - If `order` == 1:
+                        - We use K steps of DPM-Solver-1 (i.e. DDIM).
+                    - If `order` == 2:
+                        - We firstly use 1 step of DPM-Solver-1, then use (K - 1) step of multistep DPM-Solver-2.
+                    - If `order` == 3:
+                        - We firstly use 1 step of DPM-Solver-1, then 1 step of multistep DPM-Solver-2, then (K - 2) step of multistep DPM-Solver-3.
+            - 'singlestep_fixed':
+                Fixed order singlestep DPM-Solver (i.e. DPM-Solver-1 or singlestep DPM-Solver-2 or singlestep DPM-Solver-3).
+                We use singlestep DPM-Solver-`order` for `order`=1 or 2 or 3, with total [`steps` // `order`] * `order` NFE.
+            - 'adaptive':
+                Adaptive step size DPM-Solver (i.e. "DPM-Solver-12" and "DPM-Solver-23" in the paper).
+                We ignore `steps` and use adaptive step size DPM-Solver with a higher order of `order`.
+                You can adjust the absolute tolerance `atol` and the relative tolerance `rtol` to balance the computatation costs
+                (NFE) and the sample quality.
+                    - If `order` == 2, we use DPM-Solver-12 which combines DPM-Solver-1 and singlestep DPM-Solver-2.
+                    - If `order` == 3, we use DPM-Solver-23 which combines singlestep DPM-Solver-2 and singlestep DPM-Solver-3.
+
+        =====================================================
+
+        Some advices for choosing the algorithm:
+            - For **unconditional sampling** or **guided sampling with small guidance scale** by DPMs:
+                Use singlestep DPM-Solver ("DPM-Solver-fast" in the paper) with `order = 3`.
+                e.g.
+                    >>> dpm_solver = DPM_Solver(model_fn, noise_schedule, predict_x0=False)
+                    >>> x_sample = dpm_solver.sample(x, steps=steps, t_start=t_start, t_end=t_end, order=3,
+                            skip_type='time_uniform', method='singlestep')
+            - For **guided sampling with large guidance scale** by DPMs:
+                Use multistep DPM-Solver with `predict_x0 = True` and `order = 2`.
+                e.g.
+                    >>> dpm_solver = DPM_Solver(model_fn, noise_schedule, predict_x0=True)
+                    >>> x_sample = dpm_solver.sample(x, steps=steps, t_start=t_start, t_end=t_end, order=2,
+                            skip_type='time_uniform', method='multistep')
+
+        We support three types of `skip_type`:
+            - 'logSNR': uniform logSNR for the time steps. **Recommended for low-resolutional images**
+            - 'time_uniform': uniform time for the time steps. **Recommended for high-resolutional images**.
+            - 'time_quadratic': quadratic time for the time steps.
+
+        =====================================================
+        Args:
+            x: A pytorch tensor. The initial value at time `t_start`
+                e.g. if `t_start` == T, then `x` is a sample from the standard normal distribution.
+            steps: A `int`. The total number of function evaluations (NFE).
+            t_start: A `float`. The starting time of the sampling.
+                If `T` is None, we use self.noise_schedule.T (default is 1.0).
+            t_end: A `float`. The ending time of the sampling.
+                If `t_end` is None, we use 1. / self.noise_schedule.total_N.
+                e.g. if total_N == 1000, we have `t_end` == 1e-3.
+                For discrete-time DPMs:
+                    - We recommend `t_end` == 1. / self.noise_schedule.total_N.
+                For continuous-time DPMs:
+                    - We recommend `t_end` == 1e-3 when `steps` <= 15; and `t_end` == 1e-4 when `steps` > 15.
+            order: A `int`. The order of DPM-Solver.
+            skip_type: A `str`. The type for the spacing of the time steps. 'time_uniform' or 'logSNR' or 'time_quadratic'.
+            method: A `str`. The method for sampling. 'singlestep' or 'multistep' or 'singlestep_fixed' or 'adaptive'.
+            denoise: A `bool`. Whether to denoise at the final step. Default is False.
+                If `denoise` is True, the total NFE is (`steps` + 1).
+            solver_type: A `str`. The taylor expansion type for the solver. `dpm_solver` or `taylor`. We recommend `dpm_solver`.
+            atol: A `float`. The absolute tolerance of the adaptive step size solver. Valid when `method` == 'adaptive'.
+            rtol: A `float`. The relative tolerance of the adaptive step size solver. Valid when `method` == 'adaptive'.
+        Returns:
+            x_end: A pytorch tensor. The approximated solution at time `t_end`.
+
+        """
+        t_0 = 1. / self.noise_schedule.total_N if t_end is None else t_end
+        t_T = self.noise_schedule.T if t_start is None else t_start
+        device = x.device
+        if method == 'adaptive':
+            with torch.no_grad():
+                x = self.dpm_solver_adaptive(x, order=order, t_T=t_T, t_0=t_0, atol=atol, rtol=rtol,
+                                             solver_type=solver_type)
+        elif method == 'multistep':
+            assert steps >= order
+            timesteps = self.get_time_steps(skip_type=skip_type, t_T=t_T, t_0=t_0, N=steps, device=device)
+            assert timesteps.shape[0] - 1 == steps
+            with torch.no_grad():
+                vec_t = timesteps[0].expand((x.shape[0]))
+                model_prev_list = [self.model_fn(x, vec_t)]
+                t_prev_list = [vec_t]
+                # Init the first `order` values by lower order multistep DPM-Solver.
+                for init_order in range(1, order):
+                    vec_t = timesteps[init_order].expand(x.shape[0])
+                    x = self.multistep_dpm_solver_update(x, model_prev_list, t_prev_list, vec_t, init_order,
+                                                         solver_type=solver_type)
+                    model_prev_list.append(self.model_fn(x, vec_t))
+                    t_prev_list.append(vec_t)
+                # Compute the remaining values by `order`-th order multistep DPM-Solver.
+                for step in range(order, steps + 1):
+                    vec_t = timesteps[step].expand(x.shape[0])
+                    x = self.multistep_dpm_solver_update(x, model_prev_list, t_prev_list, vec_t, order,
+                                                         solver_type=solver_type)
+                    for i in range(order - 1):
+                        t_prev_list[i] = t_prev_list[i + 1]
+                        model_prev_list[i] = model_prev_list[i + 1]
+                    t_prev_list[-1] = vec_t
+                    # We do not need to evaluate the final model value.
+                    if step < steps:
+                        model_prev_list[-1] = self.model_fn(x, vec_t)
+        elif method in ['singlestep', 'singlestep_fixed']:
+            if method == 'singlestep':
+                timesteps_outer, orders = self.get_orders_and_timesteps_for_singlestep_solver(steps=steps, order=order,
+                                                                                              skip_type=skip_type,
+                                                                                              t_T=t_T, t_0=t_0,
+                                                                                              device=device)
+            elif method == 'singlestep_fixed':
+                K = steps // order
+                orders = [order, ] * K
+                timesteps_outer = self.get_time_steps(skip_type=skip_type, t_T=t_T, t_0=t_0, N=K, device=device)
+            for i, order in enumerate(orders):
+                t_T_inner, t_0_inner = timesteps_outer[i], timesteps_outer[i + 1]
+                timesteps_inner = self.get_time_steps(skip_type=skip_type, t_T=t_T_inner.item(), t_0=t_0_inner.item(),
+                                                      N=order, device=device)
+                lambda_inner = self.noise_schedule.marginal_lambda(timesteps_inner)
+                vec_s, vec_t = t_T_inner.repeat(x.shape[0]), t_0_inner.repeat(x.shape[0])
+                h = lambda_inner[-1] - lambda_inner[0]
+                r1 = None if order <= 1 else (lambda_inner[1] - lambda_inner[0]) / h
+                r2 = None if order <= 2 else (lambda_inner[2] - lambda_inner[0]) / h
+                x = self.singlestep_dpm_solver_update(x, vec_s, vec_t, order, solver_type=solver_type, r1=r1, r2=r2)
+        if denoise:
+            x = self.denoise_fn(x, torch.ones((x.shape[0],)).to(device) * t_0)
+        return x
+
+
+#############################################################
+# other utility functions
+#############################################################
+
+def interpolate_fn(x, xp, yp):
+    """
+    A piecewise linear function y = f(x), using xp and yp as keypoints.
+    We implement f(x) in a differentiable way (i.e. applicable for autograd).
+    The function f(x) is well-defined for all x-axis. (For x beyond the bounds of xp, we use the outmost points of xp to define the linear function.)
+
+    Args:
+        x: PyTorch tensor with shape [N, C], where N is the batch size, C is the number of channels (we use C = 1 for DPM-Solver).
+        xp: PyTorch tensor with shape [C, K], where K is the number of keypoints.
+        yp: PyTorch tensor with shape [C, K].
+    Returns:
+        The function values f(x), with shape [N, C].
+    """
+    N, K = x.shape[0], xp.shape[1]
+    all_x = torch.cat([x.unsqueeze(2), xp.unsqueeze(0).repeat((N, 1, 1))], dim=2)
+    sorted_all_x, x_indices = torch.sort(all_x, dim=2)
+    x_idx = torch.argmin(x_indices, dim=2)
+    cand_start_idx = x_idx - 1
+    start_idx = torch.where(
+        torch.eq(x_idx, 0),
+        torch.tensor(1, device=x.device),
+        torch.where(
+            torch.eq(x_idx, K), torch.tensor(K - 2, device=x.device), cand_start_idx,
+        ),
+    )
+    end_idx = torch.where(torch.eq(start_idx, cand_start_idx), start_idx + 2, start_idx + 1)
+    start_x = torch.gather(sorted_all_x, dim=2, index=start_idx.unsqueeze(2)).squeeze(2)
+    end_x = torch.gather(sorted_all_x, dim=2, index=end_idx.unsqueeze(2)).squeeze(2)
+    start_idx2 = torch.where(
+        torch.eq(x_idx, 0),
+        torch.tensor(0, device=x.device),
+        torch.where(
+            torch.eq(x_idx, K), torch.tensor(K - 2, device=x.device), cand_start_idx,
+        ),
+    )
+    y_positions_expanded = yp.unsqueeze(0).expand(N, -1, -1)
+    start_y = torch.gather(y_positions_expanded, dim=2, index=start_idx2.unsqueeze(2)).squeeze(2)
+    end_y = torch.gather(y_positions_expanded, dim=2, index=(start_idx2 + 1).unsqueeze(2)).squeeze(2)
+    cand = start_y + (x - start_x) * (end_y - start_y) / (end_x - start_x)
+    return cand
+
+
+def expand_dims(v, dims):
+    """
+    Expand the tensor `v` to the dim `dims`.
+
+    Args:
+        `v`: a PyTorch tensor with shape [N].
+        `dim`: a `int`.
+    Returns:
+        a PyTorch tensor with shape [N, 1, 1, ..., 1] and the total dimension is `dims`.
+    """
+    return v[(...,) + (None,) * (dims - 1)]

DDSP-SVC/diffusion/infer_gt_mel.py

@@ -0,0 +1,78 @@
+import numpy as np
+import torch
+import torch.nn.functional as F
+from diffusion.unit2mel import load_model_vocoder
+
+
+class DiffGtMel:
+    def __init__(self, project_path=None, device=None):
+        self.project_path = project_path
+        if device is not None:
+            self.device = device
+        else:
+            self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
+        self.model = None
+        self.vocoder = None
+        self.args = None
+
+    def flush_model(self, project_path, ddsp_config=None):
+        if (self.model is None) or (project_path != self.project_path):
+            model, vocoder, args = load_model_vocoder(project_path, device=self.device)
+            if self.check_args(ddsp_config, args):
+                self.model = model
+                self.vocoder = vocoder
+                self.args = args
+
+    def check_args(self, args1, args2):
+        if args1.data.block_size != args2.data.block_size:
+            raise ValueError("DDSP与DIFF模型的block_size不一致")
+        if args1.data.sampling_rate != args2.data.sampling_rate:
+            raise ValueError("DDSP与DIFF模型的sampling_rate不一致")
+        if args1.data.encoder != args2.data.encoder:
+            raise ValueError("DDSP与DIFF模型的encoder不一致")
+        return True
+
+    def __call__(self, audio, f0, hubert, volume, acc=1, spk_id=1, k_step=0, use_dpm=True,
+                 spk_mix_dict=None, start_frame=0):
+        input_mel = self.vocoder.extract(audio, self.args.data.sampling_rate)
+        if use_dpm:
+            method = 'dpm-solver'
+        else:
+            method = 'pndm'
+        out_mel = self.model(
+            hubert,
+            f0,
+            volume,
+            spk_id=spk_id,
+            spk_mix_dict=spk_mix_dict,
+            gt_spec=input_mel,
+            infer=True,
+            infer_speedup=acc,
+            method=method,
+            k_step=k_step,
+            use_tqdm=False)
+        if start_frame > 0:
+            out_mel = out_mel[:, start_frame:, :]
+            f0 = f0[:, start_frame:, :]
+        output = self.vocoder.infer(out_mel, f0)
+        if start_frame > 0:
+            output = F.pad(output, (start_frame * self.vocoder.vocoder_hop_size, 0))
+        return output
+
+    def infer(self, audio, f0, hubert, volume, acc=1, spk_id=1, k_step=0, use_dpm=True, silence_front=0,
+              use_silence=False, spk_mix_dict=None):
+        start_frame = int(silence_front * self.vocoder.vocoder_sample_rate / self.vocoder.vocoder_hop_size)
+        if use_silence:
+            audio = audio[:, start_frame * self.vocoder.vocoder_hop_size:]
+            f0 = f0[:, start_frame:, :]
+            hubert = hubert[:, start_frame:, :]
+            volume = volume[:, start_frame:, :]
+            _start_frame = 0
+        else:
+            _start_frame = start_frame
+        audio = self.__call__(audio, f0, hubert, volume, acc=acc, spk_id=spk_id, k_step=k_step,
+                              use_dpm=use_dpm, spk_mix_dict=spk_mix_dict, start_frame=_start_frame)
+        if use_silence:
+            if start_frame > 0:
+                audio = F.pad(audio, (start_frame * self.vocoder.vocoder_hop_size, 0))
+        return audio

DDSP-SVC/diffusion/solver.py

@@ -0,0 +1,171 @@
+import os
+import time
+import numpy as np
+import torch
+import librosa
+from logger.saver import Saver
+from logger import utils
+
+def test(args, model, vocoder, loader_test, saver):
+    print(' [*] testing...')
+    model.eval()
+
+    # losses
+    test_loss = 0.
+    
+    # intialization
+    num_batches = len(loader_test)
+    rtf_all = []
+    
+    # run
+    with torch.no_grad():
+        for bidx, data in enumerate(loader_test):
+            fn = data['name'][0]
+            print('--------')
+            print('{}/{} - {}'.format(bidx, num_batches, fn))
+
+            # unpack data
+            for k in data.keys():
+                if k != 'name':
+                    data[k] = data[k].to(args.device)
+            print('>>', data['name'][0])
+
+            # forward
+            st_time = time.time()
+            mel = model(
+                    data['units'], 
+                    data['f0'], 
+                    data['volume'], 
+                    data['spk_id'],
+                    gt_spec=None,
+                    infer=True, 
+                    infer_speedup=args.infer.speedup, 
+                    method=args.infer.method)
+            signal = vocoder.infer(mel, data['f0'])
+            ed_time = time.time()
+                        
+            # RTF
+            run_time = ed_time - st_time
+            song_time = signal.shape[-1] / args.data.sampling_rate
+            rtf = run_time / song_time
+            print('RTF: {}  | {} / {}'.format(rtf, run_time, song_time))
+            rtf_all.append(rtf)
+           
+            # loss
+            for i in range(args.train.batch_size):
+                loss = model(
+                    data['units'], 
+                    data['f0'], 
+                    data['volume'], 
+                    data['spk_id'], 
+                    gt_spec=data['mel'],
+                    infer=False)
+                test_loss += loss.item()
+            
+            # log mel
+            saver.log_spec(data['name'][0], data['mel'], mel)
+            
+            # log audio
+            path_audio = os.path.join(args.data.valid_path, 'audio', data['name'][0]) + '.wav'
+            audio, sr = librosa.load(path_audio, sr=args.data.sampling_rate)
+            if len(audio.shape) > 1:
+                audio = librosa.to_mono(audio)
+            audio = torch.from_numpy(audio).unsqueeze(0).to(signal)
+            saver.log_audio({fn+'/gt.wav': audio, fn+'/pred.wav': signal})
+            
+    # report
+    test_loss /= args.train.batch_size
+    test_loss /= num_batches 
+    
+    # check
+    print(' [test_loss] test_loss:', test_loss)
+    print(' Real Time Factor', np.mean(rtf_all))
+    return test_loss
+
+
+def train(args, initial_global_step, model, optimizer, scheduler, vocoder, loader_train, loader_test):
+    # saver
+    saver = Saver(args, initial_global_step=initial_global_step)
+
+    # model size
+    params_count = utils.get_network_paras_amount({'model': model})
+    saver.log_info('--- model size ---')
+    saver.log_info(params_count)
+    
+    # run
+    num_batches = len(loader_train)
+    model.train()
+    saver.log_info('======= start training =======')
+    for epoch in range(args.train.epochs):
+        for batch_idx, data in enumerate(loader_train):
+            saver.global_step_increment()
+            optimizer.zero_grad()
+
+            # unpack data
+            for k in data.keys():
+                if k != 'name':
+                    data[k] = data[k].to(args.device)
+            
+            # forward
+            loss = model(data['units'].float(), data['f0'], data['volume'], data['spk_id'], 
+                            aug_shift = data['aug_shift'], gt_spec=data['mel'].float(), infer=False)
+            
+            # handle nan loss
+            if torch.isnan(loss):
+                raise ValueError(' [x] nan loss ')
+            else:
+                # backpropagate
+                loss.backward()
+                optimizer.step()
+                scheduler.step()
+                
+            # log loss
+            if saver.global_step % args.train.interval_log == 0:
+                current_lr =  optimizer.param_groups[0]['lr']
+                saver.log_info(
+                    'epoch: {} | {:3d}/{:3d} | {} | batch/s: {:.2f} | lr: {:.6} | loss: {:.3f} | time: {} | step: {}'.format(
+                        epoch,
+                        batch_idx,
+                        num_batches,
+                        args.env.expdir,
+                        args.train.interval_log/saver.get_interval_time(),
+                        current_lr,
+                        loss.item(),
+                        saver.get_total_time(),
+                        saver.global_step
+                    )
+                )
+                
+                saver.log_value({
+                    'train/loss': loss.item()
+                })
+                
+                saver.log_value({
+                    'train/lr': current_lr
+                })
+            
+            # validation
+            if saver.global_step % args.train.interval_val == 0:
+                # save latest
+                saver.save_model(model, optimizer, postfix=f'{saver.global_step}')
+                last_val_step = saver.global_step - args.train.interval_val
+                if last_val_step % args.train.interval_force_save != 0:
+                    saver.delete_model(postfix=f'{last_val_step}')
+                
+                # run testing set
+                
+                test_loss = test(args, model, vocoder, loader_test, saver)
+             
+                saver.log_info(
+                    ' --- <validation> --- \nloss: {:.3f}. '.format(
+                        test_loss,
+                    )
+                )
+                
+                saver.log_value({
+                    'validation/loss': test_loss
+                })
+                
+                model.train()
+
+                          

DDSP-SVC/diffusion/unit2mel.py

@@ -0,0 +1,96 @@
+import os
+import yaml
+import torch
+import torch.nn as nn
+import numpy as np
+from .diffusion import GaussianDiffusion
+from .wavenet import WaveNet
+from .vocoder import Vocoder
+
+class DotDict(dict):
+    def __getattr__(*args):         
+        val = dict.get(*args)         
+        return DotDict(val) if type(val) is dict else val   
+
+    __setattr__ = dict.__setitem__    
+    __delattr__ = dict.__delitem__
+
+    
+def load_model_vocoder(
+        model_path,
+        device='cpu'):
+    config_file = os.path.join(os.path.split(model_path)[0], 'config.yaml')
+    with open(config_file, "r") as config:
+        args = yaml.safe_load(config)
+    args = DotDict(args)
+    
+    # load vocoder
+    vocoder = Vocoder(args.vocoder.type, args.vocoder.ckpt, device=device)
+    
+    # load model
+    model = Unit2Mel(
+                args.data.encoder_out_channels, 
+                args.model.n_spk,
+                args.model.use_pitch_aug,
+                vocoder.dimension,
+                args.model.n_layers,
+                args.model.n_chans,
+                args.model.n_hidden)
+    
+    print(' [Loading] ' + model_path)
+    ckpt = torch.load(model_path, map_location=torch.device(device))
+    model.to(device)
+    model.load_state_dict(ckpt['model'])
+    model.eval()
+    return model, vocoder, args
+
+
+class Unit2Mel(nn.Module):
+    def __init__(
+            self,
+            input_channel,
+            n_spk,
+            use_pitch_aug=False,
+            out_dims=128,
+            n_layers=20, 
+            n_chans=384, 
+            n_hidden=256):
+        super().__init__()
+        self.unit_embed = nn.Linear(input_channel, n_hidden)
+        self.f0_embed = nn.Linear(1, n_hidden)
+        self.volume_embed = nn.Linear(1, n_hidden)
+        if use_pitch_aug:
+            self.aug_shift_embed = nn.Linear(1, n_hidden, bias=False)
+        else:
+            self.aug_shift_embed = None
+        self.n_spk = n_spk
+        if n_spk is not None and n_spk > 1:
+            self.spk_embed = nn.Embedding(n_spk, n_hidden)
+            
+        # diffusion
+        self.decoder = GaussianDiffusion(WaveNet(out_dims, n_layers, n_chans, n_hidden), out_dims=out_dims)
+
+    def forward(self, units, f0, volume, spk_id = None, spk_mix_dict = None, aug_shift = None,
+                gt_spec=None, infer=True, infer_speedup=10, method='dpm-solver', k_step=300, use_tqdm=True):
+        
+        '''
+        input: 
+            B x n_frames x n_unit
+        return: 
+            dict of B x n_frames x feat
+        '''
+
+        x = self.unit_embed(units) + self.f0_embed((1+ f0 / 700).log()) + self.volume_embed(volume)
+        if self.n_spk is not None and self.n_spk > 1:
+            if spk_mix_dict is not None:
+                for k, v in spk_mix_dict.items():
+                    spk_id_torch = torch.LongTensor(np.array([[k]])).to(units.device)
+                    x = x + v * self.spk_embed(spk_id_torch - 1)
+            else:
+                x = x + self.spk_embed(spk_id - 1)
+        if self.aug_shift_embed is not None and aug_shift is not None:
+            x = x + self.aug_shift_embed(aug_shift / 5) 
+        x = self.decoder(x, gt_spec=gt_spec, infer=infer, infer_speedup=infer_speedup, method=method, k_step=k_step, use_tqdm=use_tqdm)
+    
+        return x
+

DDSP-SVC/diffusion/vocoder.py

@@ -0,0 +1,87 @@
+import torch
+from nsf_hifigan.nvSTFT import STFT
+from nsf_hifigan.models import load_model
+from torchaudio.transforms import Resample
+
+    
+class Vocoder:
+    def __init__(self, vocoder_type, vocoder_ckpt, device = None):
+        if device is None:
+            device = 'cuda' if torch.cuda.is_available() else 'cpu'
+        self.device = device
+        
+        if vocoder_type == 'nsf-hifigan':
+            self.vocoder = NsfHifiGAN(vocoder_ckpt, device = device)
+        elif vocoder_type == 'nsf-hifigan-log10':
+            self.vocoder = NsfHifiGANLog10(vocoder_ckpt, device = device)
+        else:
+            raise ValueError(f" [x] Unknown vocoder: {vocoder_type}")
+            
+        self.resample_kernel = {}
+        self.vocoder_sample_rate = self.vocoder.sample_rate()
+        self.vocoder_hop_size = self.vocoder.hop_size()
+        self.dimension = self.vocoder.dimension()
+        
+    def extract(self, audio, sample_rate, keyshift=0):
+                
+        # resample
+        if sample_rate == self.vocoder_sample_rate:
+            audio_res = audio
+        else:
+            key_str = str(sample_rate)
+            if key_str not in self.resample_kernel:
+                self.resample_kernel[key_str] = Resample(sample_rate, self.vocoder_sample_rate, lowpass_filter_width = 128).to(self.device)
+            audio_res = self.resample_kernel[key_str](audio)
+        
+        # extract
+        mel = self.vocoder.extract(audio_res, keyshift=keyshift) # B, n_frames, bins
+        return mel
+   
+    def infer(self, mel, f0):
+        f0 = f0[:,:mel.size(1),0] # B, n_frames
+        audio = self.vocoder(mel, f0)
+        return audio
+        
+        
+class NsfHifiGAN(torch.nn.Module):
+    def __init__(self, model_path, device=None):
+        super().__init__()
+        if device is None:
+            device = 'cuda' if torch.cuda.is_available() else 'cpu'
+        self.device = device
+        print('| Load HifiGAN: ', model_path)
+        self.model, self.h = load_model(model_path, device=self.device)
+        self.stft = STFT(
+                self.h.sampling_rate, 
+                self.h.num_mels, 
+                self.h.n_fft, 
+                self.h.win_size, 
+                self.h.hop_size, 
+                self.h.fmin, 
+                self.h.fmax)
+    
+    def sample_rate(self):
+        return self.h.sampling_rate
+        
+    def hop_size(self):
+        return self.h.hop_size
+    
+    def dimension(self):
+        return self.h.num_mels
+        
+    def extract(self, audio, keyshift=0):       
+        mel = self.stft.get_mel(audio, keyshift=keyshift).transpose(1, 2) # B, n_frames, bins
+        return mel
+    
+    def forward(self, mel, f0):
+        with torch.no_grad():
+            c = mel.transpose(1, 2)
+            audio = self.model(c, f0)
+            return audio
+
+class NsfHifiGANLog10(NsfHifiGAN):    
+    def forward(self, mel, f0):
+        with torch.no_grad():
+            c = 0.434294 * mel.transpose(1, 2)
+            audio = self.model(c, f0)
+            return audio

DDSP-SVC/diffusion/wavenet.py

@@ -0,0 +1,108 @@
+import math
+from math import sqrt
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.nn import Mish
+
+
+class Conv1d(torch.nn.Conv1d):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        nn.init.kaiming_normal_(self.weight)
+
+
+class SinusoidalPosEmb(nn.Module):
+    def __init__(self, dim):
+        super().__init__()
+        self.dim = dim
+
+    def forward(self, x):
+        device = x.device
+        half_dim = self.dim // 2
+        emb = math.log(10000) / (half_dim - 1)
+        emb = torch.exp(torch.arange(half_dim, device=device) * -emb)
+        emb = x[:, None] * emb[None, :]
+        emb = torch.cat((emb.sin(), emb.cos()), dim=-1)
+        return emb
+
+
+class ResidualBlock(nn.Module):
+    def __init__(self, encoder_hidden, residual_channels, dilation):
+        super().__init__()
+        self.residual_channels = residual_channels
+        self.dilated_conv = nn.Conv1d(
+            residual_channels,
+            2 * residual_channels,
+            kernel_size=3,
+            padding=dilation,
+            dilation=dilation
+        )
+        self.diffusion_projection = nn.Linear(residual_channels, residual_channels)
+        self.conditioner_projection = nn.Conv1d(encoder_hidden, 2 * residual_channels, 1)
+        self.output_projection = nn.Conv1d(residual_channels, 2 * residual_channels, 1)
+
+    def forward(self, x, conditioner, diffusion_step):
+        diffusion_step = self.diffusion_projection(diffusion_step).unsqueeze(-1)
+        conditioner = self.conditioner_projection(conditioner)
+        y = x + diffusion_step
+
+        y = self.dilated_conv(y) + conditioner
+
+        # Using torch.split instead of torch.chunk to avoid using onnx::Slice
+        gate, filter = torch.split(y, [self.residual_channels, self.residual_channels], dim=1)
+        y = torch.sigmoid(gate) * torch.tanh(filter)
+
+        y = self.output_projection(y)
+
+        # Using torch.split instead of torch.chunk to avoid using onnx::Slice
+        residual, skip = torch.split(y, [self.residual_channels, self.residual_channels], dim=1)
+        return (x + residual) / math.sqrt(2.0), skip
+
+
+class WaveNet(nn.Module):
+    def __init__(self, in_dims=128, n_layers=20, n_chans=384, n_hidden=256):
+        super().__init__()
+        self.input_projection = Conv1d(in_dims, n_chans, 1)
+        self.diffusion_embedding = SinusoidalPosEmb(n_chans)
+        self.mlp = nn.Sequential(
+            nn.Linear(n_chans, n_chans * 4),
+            Mish(),
+            nn.Linear(n_chans * 4, n_chans)
+        )
+        self.residual_layers = nn.ModuleList([
+            ResidualBlock(
+                encoder_hidden=n_hidden,
+                residual_channels=n_chans,
+                dilation=1
+            )
+            for i in range(n_layers)
+        ])
+        self.skip_projection = Conv1d(n_chans, n_chans, 1)
+        self.output_projection = Conv1d(n_chans, in_dims, 1)
+        nn.init.zeros_(self.output_projection.weight)
+
+    def forward(self, spec, diffusion_step, cond):
+        """
+        :param spec: [B, 1, M, T]
+        :param diffusion_step: [B, 1]
+        :param cond: [B, M, T]
+        :return:
+        """
+        x = spec.squeeze(1)
+        x = self.input_projection(x)  # [B, residual_channel, T]
+
+        x = F.relu(x)
+        diffusion_step = self.diffusion_embedding(diffusion_step)
+        diffusion_step = self.mlp(diffusion_step)
+        skip = []
+        for layer in self.residual_layers:
+            x, skip_connection = layer(x, cond, diffusion_step)
+            skip.append(skip_connection)
+
+        x = torch.sum(torch.stack(skip), dim=0) / sqrt(len(self.residual_layers))
+        x = self.skip_projection(x)
+        x = F.relu(x)
+        x = self.output_projection(x)  # [B, mel_bins, T]
+        return x[:, None, :, :]

DDSP-SVC/draw.py

@@ -0,0 +1,102 @@
+import numpy as np
+import tqdm
+import matplotlib.pyplot as plt
+import os
+import shutil
+import wave
+
+WAV_MIN_LENGTH = 2    # wav文件的最短时长 / The minimum duration of wav files
+SAMPLE_RATE = 1    # 抽取文件数量的百分比 / The percentage of files to be extracted
+SAMPLE_MIN = 2    # 抽取的文件数量下限 / The lower limit of the number of files to be extracted
+SAMPLE_MAX = 10    # 抽取的文件数量上限 / The upper limit of the number of files to be extracted
+
+
+# 定义一个函数，用于检查wav文件的时长是否大于最短时长
+def check_duration(wav_file):
+    # 打开wav文件
+    f = wave.open(wav_file, "rb")
+    # 获取帧数和帧率
+    frames = f.getnframes()
+    rate = f.getframerate()
+    # 计算时长（秒）
+    duration = frames / float(rate)
+    # 关闭文件
+    f.close()
+    # 返回时长是否大于最短时长的布尔值
+    return duration > WAV_MIN_LENGTH
+
+# 定义一个函数，用于从给定的目录中随机抽取一定比例的wav文件，并剪切到另一个目录中，保留数据结构
+def split_data(src_dir, dst_dir, ratio):
+    # 创建目标目录（如果不存在）
+    if not os.path.exists(dst_dir):
+        os.makedirs(dst_dir)
+    
+    # 获取源目录下所有的子目录和文件名
+    subdirs, files, subfiles = [], [], []
+    for item in os.listdir(src_dir):
+        item_path = os.path.join(src_dir, item)
+        if os.path.isdir(item_path):
+            subdirs.append(item)
+            for subitem in os.listdir(item_path):
+                subitem_path = os.path.join(item_path, subitem)
+                if os.path.isfile(subitem_path) and subitem.endswith(".wav"):
+                    subfiles.append(subitem)
+        elif os.path.isfile(item_path) and item.endswith(".wav"):
+            files.append(item)
+
+    # 如果源目录下没有任何wav文件，则报错并退出函数
+    if len(files) == 0:
+        if len(subfiles) == 0:
+            print(f"Error: No wav files found in {src_dir}")
+            return
+    
+    # 计算需要抽取的wav文件数量
+    num_files = int(len(files) * ratio)
+    num_files = max(SAMPLE_MIN, min(SAMPLE_MAX, num_files))
+
+    # 随机打乱文件名列表，并取出前num_files个作为抽取结果
+    np.random.shuffle(files)
+    selected_files = files[:num_files]
+    
+    # 创建一个进度条对象，用于显示程序的运行进度
+    pbar = tqdm.tqdm(total=num_files)
+
+    # 遍历抽取结果中的每个文件名，检查是否大于2秒
+    for file in selected_files:
+        src_file = os.path.join(src_dir, file)
+        # 检查源文件的时长是否大于2秒，如果不是，则打印源文件的文件名，并跳过该文件
+        if not check_duration(src_file):
+            print(f"Skipped {src_file} because its duration is less than 2 seconds.")
+            continue
+        # 拼接源文件和目标文件的完整路径，移动文件，并更新进度条
+        dst_file = os.path.join(dst_dir, file)
+        shutil.move(src_file, dst_file)
+        pbar.update(1)
+
+    pbar.close()
+
+    # 遍历源目录下所有的子目录（如果有）
+    for subdir in subdirs:
+        # 拼接子目录在源目录和目标目录中的完整路径
+        src_subdir = os.path.join(src_dir, subdir)
+        dst_subdir = os.path.join(dst_dir, subdir)
+        # 递归地调用本函数，对子目录中的wav文件进行同样的操作，保留数据结构
+        split_data(src_subdir, dst_subdir, ratio)
+
+# 定义主函数，用于获取用户输入并调用上述函数
+
+def main():
+    root_dir = os.path.abspath('.')
+    dst_dir = root_dir + "/data/val/audio"
+    # 抽取比例，默认为1
+    ratio = float(SAMPLE_RATE) / 100
+
+    # 固定源目录为根目录下/data/train/audio目录
+    src_dir = root_dir + "/data/train/audio"
+
+    # 调用split_data函数，对源目录中的wav文件进行抽取，并剪切到目标目录中，保留数据结构
+    split_data(src_dir, dst_dir, ratio)
+
+# 如果本模块是主模块，则执行主函数
+if __name__ == "__main__":
+    main()

DDSP-SVC/encoder/hubert/model.py

@@ -0,0 +1,293 @@
+import copy
+from typing import Optional, Tuple
+import random
+
+from sklearn.cluster import KMeans
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.nn.modules.utils import consume_prefix_in_state_dict_if_present
+
+URLS = {
+    "hubert-discrete": "https://github.com/bshall/hubert/releases/download/v0.1/hubert-discrete-e9416457.pt",
+    "hubert-soft": "https://github.com/bshall/hubert/releases/download/v0.1/hubert-soft-0d54a1f4.pt",
+    "kmeans100": "https://github.com/bshall/hubert/releases/download/v0.1/kmeans100-50f36a95.pt",
+}
+
+
+class Hubert(nn.Module):
+    def __init__(self, num_label_embeddings: int = 100, mask: bool = True):
+        super().__init__()
+        self._mask = mask
+        self.feature_extractor = FeatureExtractor()
+        self.feature_projection = FeatureProjection()
+        self.positional_embedding = PositionalConvEmbedding()
+        self.norm = nn.LayerNorm(768)
+        self.dropout = nn.Dropout(0.1)
+        self.encoder = TransformerEncoder(
+            nn.TransformerEncoderLayer(
+                768, 12, 3072, activation="gelu", batch_first=True
+            ),
+            12,
+        )
+        self.proj = nn.Linear(768, 256)
+
+        self.masked_spec_embed = nn.Parameter(torch.FloatTensor(768).uniform_())
+        self.label_embedding = nn.Embedding(num_label_embeddings, 256)
+
+    def mask(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+        mask = None
+        if self.training and self._mask:
+            mask = _compute_mask((x.size(0), x.size(1)), 0.8, 10, x.device, 2)
+            x[mask] = self.masked_spec_embed.to(x.dtype)
+        return x, mask
+
+    def encode(
+        self, x: torch.Tensor, layer: Optional[int] = None
+    ) -> Tuple[torch.Tensor, torch.Tensor]:
+        x = self.feature_extractor(x)
+        x = self.feature_projection(x.transpose(1, 2))
+        x, mask = self.mask(x)
+        x = x + self.positional_embedding(x)
+        x = self.dropout(self.norm(x))
+        x = self.encoder(x, output_layer=layer)
+        return x, mask
+
+    def logits(self, x: torch.Tensor) -> torch.Tensor:
+        logits = torch.cosine_similarity(
+            x.unsqueeze(2),
+            self.label_embedding.weight.unsqueeze(0).unsqueeze(0),
+            dim=-1,
+        )
+        return logits / 0.1
+
+    def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+        x, mask = self.encode(x)
+        x = self.proj(x)
+        logits = self.logits(x)
+        return logits, mask
+
+
+class HubertSoft(Hubert):
+    def __init__(self):
+        super().__init__()
+
+    @torch.inference_mode()
+    def units(self, wav: torch.Tensor) -> torch.Tensor:
+        wav = F.pad(wav, ((400 - 320) // 2, (400 - 320) // 2))
+        x, _ = self.encode(wav)
+        return self.proj(x)
+
+
+class HubertDiscrete(Hubert):
+    def __init__(self, kmeans):
+        super().__init__(504)
+        self.kmeans = kmeans
+
+    @torch.inference_mode()
+    def units(self, wav: torch.Tensor) -> torch.LongTensor:
+        wav = F.pad(wav, ((400 - 320) // 2, (400 - 320) // 2))
+        x, _ = self.encode(wav, layer=7)
+        x = self.kmeans.predict(x.squeeze().cpu().numpy())
+        return torch.tensor(x, dtype=torch.long, device=wav.device)
+
+
+class FeatureExtractor(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.conv0 = nn.Conv1d(1, 512, 10, 5, bias=False)
+        self.norm0 = nn.GroupNorm(512, 512)
+        self.conv1 = nn.Conv1d(512, 512, 3, 2, bias=False)
+        self.conv2 = nn.Conv1d(512, 512, 3, 2, bias=False)
+        self.conv3 = nn.Conv1d(512, 512, 3, 2, bias=False)
+        self.conv4 = nn.Conv1d(512, 512, 3, 2, bias=False)
+        self.conv5 = nn.Conv1d(512, 512, 2, 2, bias=False)
+        self.conv6 = nn.Conv1d(512, 512, 2, 2, bias=False)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = F.gelu(self.norm0(self.conv0(x)))
+        x = F.gelu(self.conv1(x))
+        x = F.gelu(self.conv2(x))
+        x = F.gelu(self.conv3(x))
+        x = F.gelu(self.conv4(x))
+        x = F.gelu(self.conv5(x))
+        x = F.gelu(self.conv6(x))
+        return x
+
+
+class FeatureProjection(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.norm = nn.LayerNorm(512)
+        self.projection = nn.Linear(512, 768)
+        self.dropout = nn.Dropout(0.1)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.norm(x)
+        x = self.projection(x)
+        x = self.dropout(x)
+        return x
+
+
+class PositionalConvEmbedding(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.conv = nn.Conv1d(
+            768,
+            768,
+            kernel_size=128,
+            padding=128 // 2,
+            groups=16,
+        )
+        self.conv = nn.utils.weight_norm(self.conv, name="weight", dim=2)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.conv(x.transpose(1, 2))
+        x = F.gelu(x[:, :, :-1])
+        return x.transpose(1, 2)
+
+
+class TransformerEncoder(nn.Module):
+    def __init__(
+        self, encoder_layer: nn.TransformerEncoderLayer, num_layers: int
+    ) -> None:
+        super(TransformerEncoder, self).__init__()
+        self.layers = nn.ModuleList(
+            [copy.deepcopy(encoder_layer) for _ in range(num_layers)]
+        )
+        self.num_layers = num_layers
+
+    def forward(
+        self,
+        src: torch.Tensor,
+        mask: torch.Tensor = None,
+        src_key_padding_mask: torch.Tensor = None,
+        output_layer: Optional[int] = None,
+    ) -> torch.Tensor:
+        output = src
+        for layer in self.layers[:output_layer]:
+            output = layer(
+                output, src_mask=mask, src_key_padding_mask=src_key_padding_mask
+            )
+        return output
+
+
+def _compute_mask(
+    shape: Tuple[int, int],
+    mask_prob: float,
+    mask_length: int,
+    device: torch.device,
+    min_masks: int = 0,
+) -> torch.Tensor:
+    batch_size, sequence_length = shape
+
+    if mask_length < 1:
+        raise ValueError("`mask_length` has to be bigger than 0.")
+
+    if mask_length > sequence_length:
+        raise ValueError(
+            f"`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: {mask_length} and `sequence_length`: {sequence_length}`"
+        )
+
+    # compute number of masked spans in batch
+    num_masked_spans = int(mask_prob * sequence_length / mask_length + random.random())
+    num_masked_spans = max(num_masked_spans, min_masks)
+
+    # make sure num masked indices <= sequence_length
+    if num_masked_spans * mask_length > sequence_length:
+        num_masked_spans = sequence_length // mask_length
+
+    # SpecAugment mask to fill
+    mask = torch.zeros((batch_size, sequence_length), device=device, dtype=torch.bool)
+
+    # uniform distribution to sample from, make sure that offset samples are < sequence_length
+    uniform_dist = torch.ones(
+        (batch_size, sequence_length - (mask_length - 1)), device=device
+    )
+
+    # get random indices to mask
+    mask_indices = torch.multinomial(uniform_dist, num_masked_spans)
+
+    # expand masked indices to masked spans
+    mask_indices = (
+        mask_indices.unsqueeze(dim=-1)
+        .expand((batch_size, num_masked_spans, mask_length))
+        .reshape(batch_size, num_masked_spans * mask_length)
+    )
+    offsets = (
+        torch.arange(mask_length, device=device)[None, None, :]
+        .expand((batch_size, num_masked_spans, mask_length))
+        .reshape(batch_size, num_masked_spans * mask_length)
+    )
+    mask_idxs = mask_indices + offsets
+
+    # scatter indices to mask
+    mask = mask.scatter(1, mask_idxs, True)
+
+    return mask
+
+
+def hubert_discrete(
+    pretrained: bool = True,
+    progress: bool = True,
+) -> HubertDiscrete:
+    r"""HuBERT-Discrete from `"A Comparison of Discrete and Soft Speech Units for Improved Voice Conversion"`.
+    Args:
+        pretrained (bool): load pretrained weights into the model
+        progress (bool): show progress bar when downloading model
+    """
+    kmeans = kmeans100(pretrained=pretrained, progress=progress)
+    hubert = HubertDiscrete(kmeans)
+    if pretrained:
+        checkpoint = torch.hub.load_state_dict_from_url(
+            URLS["hubert-discrete"], progress=progress
+        )
+        consume_prefix_in_state_dict_if_present(checkpoint, "module.")
+        hubert.load_state_dict(checkpoint)
+        hubert.eval()
+    return hubert
+
+
+def hubert_soft(
+    pretrained: bool = True,
+    progress: bool = True,
+) -> HubertSoft:
+    r"""HuBERT-Soft from `"A Comparison of Discrete and Soft Speech Units for Improved Voice Conversion"`.
+    Args:
+        pretrained (bool): load pretrained weights into the model
+        progress (bool): show progress bar when downloading model
+    """
+    hubert = HubertSoft()
+    if pretrained:
+        checkpoint = torch.hub.load_state_dict_from_url(
+            URLS["hubert-soft"], progress=progress
+        )
+        consume_prefix_in_state_dict_if_present(checkpoint, "module.")
+        hubert.load_state_dict(checkpoint)
+        hubert.eval()
+    return hubert
+
+
+def _kmeans(
+    num_clusters: int, pretrained: bool = True, progress: bool = True
+) -> KMeans:
+    kmeans = KMeans(num_clusters)
+    if pretrained:
+        checkpoint = torch.hub.load_state_dict_from_url(
+            URLS[f"kmeans{num_clusters}"], progress=progress
+        )
+        kmeans.__dict__["n_features_in_"] = checkpoint["n_features_in_"]
+        kmeans.__dict__["_n_threads"] = checkpoint["_n_threads"]
+        kmeans.__dict__["cluster_centers_"] = checkpoint["cluster_centers_"].numpy()
+    return kmeans
+
+
+def kmeans100(pretrained: bool = True, progress: bool = True) -> KMeans:
+    r"""
+    k-means checkpoint for HuBERT-Discrete with 100 clusters.
+    Args:
+        pretrained (bool): load pretrained weights into the model
+        progress (bool): show progress bar when downloading model
+    """
+    return _kmeans(100, pretrained, progress)

DDSP-SVC/enhancer.py

@@ -0,0 +1,115 @@
+import numpy as np
+import torch
+import torch.nn.functional as F
+from nsf_hifigan.nvSTFT import STFT
+from nsf_hifigan.models import load_model
+from torchaudio.transforms import Resample
+
+class Enhancer:
+    def __init__(self, enhancer_type, enhancer_ckpt, device=None):
+        if device is None:
+            device = 'cuda' if torch.cuda.is_available() else 'cpu'
+        self.device = device
+        
+        if enhancer_type == 'nsf-hifigan':
+            self.enhancer = NsfHifiGAN(enhancer_ckpt, device=self.device)
+        else:
+            raise ValueError(f" [x] Unknown enhancer: {enhancer_type}")
+        
+        self.resample_kernel = {}
+        self.enhancer_sample_rate = self.enhancer.sample_rate()
+        self.enhancer_hop_size = self.enhancer.hop_size()
+        
+    def enhance(self,
+                audio, # 1, T
+                sample_rate,
+                f0, # 1, n_frames, 1
+                hop_size,
+                adaptive_key = 0,
+                silence_front = 0
+                ):
+        # enhancer start time 
+        start_frame = int(silence_front * sample_rate / hop_size)
+        real_silence_front = start_frame * hop_size / sample_rate
+        audio = audio[:, int(np.round(real_silence_front * sample_rate)) : ]
+        f0 = f0[: , start_frame :, :]
+        
+        # adaptive parameters
+        if adaptive_key == 'auto':
+            adaptive_key = 12 * np.log2(float(torch.max(f0) / 760))
+            adaptive_key = max(0, np.ceil(adaptive_key))
+            print('auto_adaptive_key: ' + str(int(adaptive_key)))
+        else:
+            adaptive_key = float(adaptive_key)
+        
+        adaptive_factor = 2 ** ( -adaptive_key / 12)    
+        adaptive_sample_rate = 100 * int(np.round(self.enhancer_sample_rate / adaptive_factor / 100))
+        real_factor = self.enhancer_sample_rate / adaptive_sample_rate
+        
+        # resample the ddsp output
+        if sample_rate == adaptive_sample_rate:
+            audio_res = audio
+        else:
+            key_str = str(sample_rate) + str(adaptive_sample_rate)
+            if key_str not in self.resample_kernel:
+                self.resample_kernel[key_str] = Resample(sample_rate, adaptive_sample_rate, lowpass_filter_width = 128).to(self.device)
+            audio_res = self.resample_kernel[key_str](audio)
+        
+        n_frames = int(audio_res.size(-1) // self.enhancer_hop_size + 1)
+        
+        # resample f0
+        if hop_size == self.enhancer_hop_size and sample_rate == self.enhancer_sample_rate and sample_rate == adaptive_sample_rate:
+            f0_res = f0.squeeze(-1) # 1, n_frames
+        else:
+            f0_np = f0.squeeze(0).squeeze(-1).cpu().numpy()
+            f0_np *= real_factor
+            time_org = (hop_size / sample_rate) * np.arange(len(f0_np)) / real_factor
+            time_frame = (self.enhancer_hop_size / self.enhancer_sample_rate) * np.arange(n_frames)
+            f0_res = np.interp(time_frame, time_org, f0_np, left=f0_np[0], right=f0_np[-1])
+            f0_res = torch.from_numpy(f0_res).unsqueeze(0).float().to(self.device) # 1, n_frames
+
+        # enhance
+        enhanced_audio, enhancer_sample_rate = self.enhancer(audio_res, f0_res)
+        
+        # resample the enhanced output
+        if adaptive_sample_rate != enhancer_sample_rate:
+            key_str = str(adaptive_sample_rate) + str(enhancer_sample_rate)
+            if key_str not in self.resample_kernel:
+                self.resample_kernel[key_str] = Resample(adaptive_sample_rate, enhancer_sample_rate, lowpass_filter_width = 128).to(self.device)
+            enhanced_audio =  self.resample_kernel[key_str](enhanced_audio)
+        
+        # pad the silence frames
+        if start_frame > 0:
+            enhanced_audio = F.pad(enhanced_audio, (int(np.round(enhancer_sample_rate * real_silence_front)), 0))
+            
+        return enhanced_audio, enhancer_sample_rate
+        
+        
+class NsfHifiGAN(torch.nn.Module):
+    def __init__(self, model_path, device=None):
+        super().__init__()
+        if device is None:
+            device = 'cuda' if torch.cuda.is_available() else 'cpu'
+        self.device = device
+        print('| Load HifiGAN: ', model_path)
+        self.model, self.h = load_model(model_path, device=self.device)
+        self.stft = STFT(
+                self.h.sampling_rate, 
+                self.h.num_mels, 
+                self.h.n_fft, 
+                self.h.win_size, 
+                self.h.hop_size, 
+                self.h.fmin, 
+                self.h.fmax)
+    
+    def sample_rate(self):
+        return self.h.sampling_rate
+        
+    def hop_size(self):
+        return self.h.hop_size
+        
+    def forward(self, audio, f0):
+        with torch.no_grad():
+            mel = self.stft.get_mel(audio)
+            enhanced_audio = self.model(mel, f0[:,:mel.size(-1)])
+            return enhanced_audio, self.h.sampling_rate

DDSP-SVC/exp/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

DDSP-SVC/flask_api.py

@@ -0,0 +1,178 @@
+import io
+import logging
+import torch
+import numpy as np
+import slicer
+import soundfile as sf
+import librosa
+from flask import Flask, request, send_file
+from flask_cors import CORS
+
+from ddsp.vocoder import load_model, F0_Extractor, Volume_Extractor, Units_Encoder
+from ddsp.core import upsample
+from enhancer import Enhancer
+
+
+app = Flask(__name__)
+
+CORS(app)
+
+logging.getLogger("numba").setLevel(logging.WARNING)
+
+
+@app.route("/voiceChangeModel", methods=["POST"])
+def voice_change_model():
+    request_form = request.form
+    wave_file = request.files.get("sample", None)
+    # get fSafePrefixPadLength
+    f_safe_prefix_pad_length = float(request_form.get("fSafePrefixPadLength", 0))
+    print("f_safe_prefix_pad_length:"+str(f_safe_prefix_pad_length))
+    # 变调信息
+    f_pitch_change = float(request_form.get("fPitchChange", 0))
+    # 获取spk_id
+    int_speak_id = int(request_form.get("sSpeakId", 0))
+    if enable_spk_id_cover:
+        int_speak_id = spk_id
+    # print("说话人:" + str(int_speak_id))
+    # DAW所需的采样率
+    daw_sample = int(float(request_form.get("sampleRate", 0)))
+    # http获得wav文件并转换
+    input_wav_read = io.BytesIO(wave_file.read())
+    # 模型推理
+    _audio, _model_sr = svc_model.infer(input_wav_read, f_pitch_change, int_speak_id, f_safe_prefix_pad_length)
+    tar_audio = librosa.resample(_audio, _model_sr, daw_sample)
+    # 返回音频
+    out_wav_path = io.BytesIO()
+    sf.write(out_wav_path, tar_audio, daw_sample, format="wav")
+    out_wav_path.seek(0)
+    return send_file(out_wav_path, download_name="temp.wav", as_attachment=True)
+
+
+class SvcDDSP:
+    def __init__(self, model_path, vocoder_based_enhancer, enhancer_adaptive_key, input_pitch_extractor,
+                 f0_min, f0_max, threhold, spk_id, spk_mix_dict, enable_spk_id_cover):
+        self.model_path = model_path
+        self.vocoder_based_enhancer = vocoder_based_enhancer
+        self.enhancer_adaptive_key = enhancer_adaptive_key
+        self.input_pitch_extractor = input_pitch_extractor
+        self.f0_min = f0_min
+        self.f0_max = f0_max
+        self.threhold = threhold
+        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
+        self.spk_id = spk_id
+        self.spk_mix_dict = spk_mix_dict
+        self.enable_spk_id_cover = enable_spk_id_cover
+        
+        # load ddsp model
+        self.model, self.args = load_model(self.model_path, device=self.device)
+        
+        # load units encoder
+        if self.args.data.encoder == 'cnhubertsoftfish':
+            cnhubertsoft_gate = self.args.data.cnhubertsoft_gate
+        else:
+            cnhubertsoft_gate = 10
+        self.units_encoder = Units_Encoder(
+            self.args.data.encoder,
+            self.args.data.encoder_ckpt,
+            self.args.data.encoder_sample_rate,
+            self.args.data.encoder_hop_size,
+            cnhubertsoft_gate=cnhubertsoft_gate,
+            device=self.device)
+        
+        # load enhancer
+        if self.vocoder_based_enhancer:
+            self.enhancer = Enhancer(self.args.enhancer.type, self.args.enhancer.ckpt, device=self.device)
+
+    def infer(self, input_wav, pitch_adjust, speaker_id, safe_prefix_pad_length):
+        print("Infer!")
+        # load input
+        audio, sample_rate = librosa.load(input_wav, sr=None, mono=True)
+        if len(audio.shape) > 1:
+            audio = librosa.to_mono(audio)
+        hop_size = self.args.data.block_size * sample_rate / self.args.data.sampling_rate
+        
+        # safe front silence
+        if safe_prefix_pad_length > 0.03:
+            silence_front = safe_prefix_pad_length - 0.03
+        else:
+            silence_front = 0
+            
+        # extract f0
+        pitch_extractor = F0_Extractor(
+            self.input_pitch_extractor,
+            sample_rate,
+            hop_size,
+            float(self.f0_min),
+            float(self.f0_max))
+        f0 = pitch_extractor.extract(audio, uv_interp=True, device=self.device, silence_front=silence_front)
+        f0 = torch.from_numpy(f0).float().to(self.device).unsqueeze(-1).unsqueeze(0)
+        f0 = f0 * 2 ** (float(pitch_adjust) / 12)
+        
+        # extract volume
+        volume_extractor = Volume_Extractor(hop_size)
+        volume = volume_extractor.extract(audio)
+        mask = (volume > 10 ** (float(self.threhold) / 20)).astype('float')
+        mask = np.pad(mask, (4, 4), constant_values=(mask[0], mask[-1]))
+        mask = np.array([np.max(mask[n : n + 9]) for n in range(len(mask) - 8)])
+        mask = torch.from_numpy(mask).float().to(self.device).unsqueeze(-1).unsqueeze(0)
+        mask = upsample(mask, self.args.data.block_size).squeeze(-1)
+        volume = torch.from_numpy(volume).float().to(self.device).unsqueeze(-1).unsqueeze(0)
+
+        # extract units
+        audio_t = torch.from_numpy(audio).float().unsqueeze(0).to(self.device)
+        units = self.units_encoder.encode(audio_t, sample_rate, hop_size)
+        
+        # spk_id or spk_mix_dict
+        if self.enable_spk_id_cover:
+            spk_id = self.spk_id
+        else:
+            spk_id = speaker_id
+        spk_id = torch.LongTensor(np.array([[spk_id]])).to(self.device)
+        
+        # forward and return the output
+        with torch.no_grad():
+            output, _, (s_h, s_n) = self.model(units, f0, volume, spk_id = spk_id, spk_mix_dict = self.spk_mix_dict)
+            output *= mask
+            if self.vocoder_based_enhancer:
+                output, output_sample_rate = self.enhancer.enhance(
+                                                                output, 
+                                                                self.args.data.sampling_rate, 
+                                                                f0, 
+                                                                self.args.data.block_size,
+                                                                adaptive_key = self.enhancer_adaptive_key,
+                                                                silence_front = silence_front)
+            else:
+                output_sample_rate = self.args.data.sampling_rate
+
+            output = output.squeeze().cpu().numpy()
+            return output, output_sample_rate
+
+
+if __name__ == "__main__":
+    # ddsp-svc下只需传入下列参数。
+    # 对接的是串串香火锅大佬https://github.com/zhaohui8969/VST_NetProcess-。建议使用最新版本。
+    # flask部分来自diffsvc小狼大佬编写的代码。
+    # config和模型得同一目录。
+    checkpoint_path = "exp/multi_speaker/model_300000.pt"
+    # 是否使用预训练的基于声码器的增强器增强输出，但对硬件要求更高。
+    use_vocoder_based_enhancer = True
+    # 结合增强器使用，0为正常音域范围（最高G5)内的高音频质量，大于0则可以防止超高音破音
+    enhancer_adaptive_key = 0
+    # f0提取器，有parselmouth, dio, harvest, crepe
+    select_pitch_extractor = 'crepe'
+    # f0范围限制(Hz)
+    limit_f0_min = 50
+    limit_f0_max = 1100
+    # 音量响应阈值(dB)
+    threhold = -60
+    # 默认说话人。以及是否优先使用默认说话人覆盖vst传入的参数。
+    spk_id = 1
+    enable_spk_id_cover = True
+    # 混合说话人字典（捏音色功能）
+    # 设置为非 None 字典会覆盖 spk_id
+    spk_mix_dict = None # {1:0.5, 2:0.5} 表示1号说话人和2号说话人的音色按照0.5:0.5的比例混合
+    svc_model = SvcDDSP(checkpoint_path, use_vocoder_based_enhancer, enhancer_adaptive_key, select_pitch_extractor,
+                        limit_f0_min, limit_f0_max, threhold, spk_id, spk_mix_dict, enable_spk_id_cover)
+
+    # 此处与vst插件对应，端口必须接上。
+    app.run(port=6844, host="0.0.0.0", debug=False, threaded=False)

DDSP-SVC/gui.py

@@ -0,0 +1,483 @@
+import PySimpleGUI as sg
+import sounddevice as sd
+import torch, librosa, threading, pickle
+from enhancer import Enhancer
+import numpy as np
+from torch.nn import functional as F
+from torchaudio.transforms import Resample
+from ddsp.vocoder import load_model, F0_Extractor, Volume_Extractor, Units_Encoder
+from ddsp.core import upsample
+import time
+import gui_locale
+
+
+def phase_vocoder(a, b, fade_out, fade_in):
+    fa = torch.fft.rfft(a)
+    fb = torch.fft.rfft(b)
+    absab = torch.abs(fa) + torch.abs(fb)
+    n = a.shape[0]
+    if n % 2 == 0:
+        absab[1:-1] *= 2
+    else:
+        absab[1:] *= 2
+    phia = torch.angle(fa)
+    phib = torch.angle(fb)
+    deltaphase = phib - phia
+    deltaphase = deltaphase - 2 * np.pi * torch.floor(deltaphase / 2 / np.pi + 0.5)
+    w = 2 * np.pi * torch.arange(n // 2 + 1).to(a) + deltaphase
+    t = torch.arange(n).unsqueeze(-1).to(a) / n
+    result = a * (fade_out ** 2) + b * (fade_in ** 2) + torch.sum(absab * torch.cos(w * t + phia),
+                                                                  -1) * fade_out * fade_in / n
+    return result
+
+
+class SvcDDSP:
+    def __init__(self) -> None:
+        self.model = None
+        self.units_encoder = None
+        self.encoder_type = None
+        self.encoder_ckpt = None
+        self.enhancer = None
+        self.enhancer_type = None
+        self.enhancer_ckpt = None
+
+    def update_model(self, model_path):
+        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+        # load ddsp model
+        if self.model is None or self.model_path != model_path:
+            self.model, self.args = load_model(model_path, device=self.device)
+            self.model_path = model_path
+
+            # load units encoder
+            if self.units_encoder is None or self.args.data.encoder != self.encoder_type or self.args.data.encoder_ckpt != self.encoder_ckpt:
+                if self.args.data.encoder == 'cnhubertsoftfish':
+                    cnhubertsoft_gate = self.args.data.cnhubertsoft_gate
+                else:
+                    cnhubertsoft_gate = 10
+                self.units_encoder = Units_Encoder(
+                    self.args.data.encoder,
+                    self.args.data.encoder_ckpt,
+                    self.args.data.encoder_sample_rate,
+                    self.args.data.encoder_hop_size,
+                    cnhubertsoft_gate=cnhubertsoft_gate,
+                    device=self.device)
+                self.encoder_type = self.args.data.encoder
+                self.encoder_ckpt = self.args.data.encoder_ckpt
+
+        # load enhancer
+        if self.enhancer is None or self.args.enhancer.type != self.enhancer_type or self.args.enhancer.ckpt != self.enhancer_ckpt:
+            self.enhancer = Enhancer(self.args.enhancer.type, self.args.enhancer.ckpt, device=self.device)
+            self.enhancer_type = self.args.enhancer.type
+            self.enhancer_ckpt = self.args.enhancer.ckpt
+
+    def infer(self,
+              audio,
+              sample_rate,
+              spk_id=1,
+              threhold=-45,
+              pitch_adjust=0,
+              use_spk_mix=False,
+              spk_mix_dict=None,
+              use_enhancer=True,
+              enhancer_adaptive_key='auto',
+              pitch_extractor_type='crepe',
+              f0_min=50,
+              f0_max=1100,
+              safe_prefix_pad_length=0,
+              ):
+        print("Infering...")
+        # load input
+        # audio, sample_rate = librosa.load(input_wav, sr=None, mono=True)
+        hop_size = self.args.data.block_size * sample_rate / self.args.data.sampling_rate
+        # safe front silence
+        if safe_prefix_pad_length > 0.03:
+            silence_front = safe_prefix_pad_length - 0.03
+        else:
+            silence_front = 0
+
+        # extract f0
+        pitch_extractor = F0_Extractor(
+            pitch_extractor_type,
+            sample_rate,
+            hop_size,
+            float(f0_min),
+            float(f0_max))
+        f0 = pitch_extractor.extract(audio, uv_interp=True, device=self.device, silence_front=silence_front)
+        f0 = torch.from_numpy(f0).float().to(self.device).unsqueeze(-1).unsqueeze(0)
+        f0 = f0 * 2 ** (float(pitch_adjust) / 12)
+
+        # extract volume
+        volume_extractor = Volume_Extractor(hop_size)
+        volume = volume_extractor.extract(audio)
+        mask = (volume > 10 ** (float(threhold) / 20)).astype('float')
+        mask = np.pad(mask, (4, 4), constant_values=(mask[0], mask[-1]))
+        mask = np.array([np.max(mask[n: n + 9]) for n in range(len(mask) - 8)])
+        mask = torch.from_numpy(mask).float().to(self.device).unsqueeze(-1).unsqueeze(0)
+        mask = upsample(mask, self.args.data.block_size).squeeze(-1)
+        volume = torch.from_numpy(volume).float().to(self.device).unsqueeze(-1).unsqueeze(0)
+
+        # extract units
+        audio_t = torch.from_numpy(audio).float().unsqueeze(0).to(self.device)
+        units = self.units_encoder.encode(audio_t, sample_rate, hop_size)
+
+        # spk_id or spk_mix_dict
+        spk_id = torch.LongTensor(np.array([[spk_id]])).to(self.device)
+        dictionary = None
+        if use_spk_mix:
+            dictionary = spk_mix_dict
+
+            # forward and return the output
+        with torch.no_grad():
+            output, _, (s_h, s_n) = self.model(units, f0, volume, spk_id=spk_id, spk_mix_dict=dictionary)
+            output *= mask
+            if use_enhancer:
+                output, output_sample_rate = self.enhancer.enhance(
+                    output,
+                    self.args.data.sampling_rate,
+                    f0,
+                    self.args.data.block_size,
+                    adaptive_key=enhancer_adaptive_key,
+                    silence_front=silence_front)
+            else:
+                output_sample_rate = self.args.data.sampling_rate
+
+            output = output.squeeze()
+            return output, output_sample_rate
+
+
+class Config:
+    def __init__(self) -> None:
+        self.samplerate = 44100  # Hz
+        self.block_time = 1.5  # s
+        self.f_pitch_change: float = 0.0  # float(request_form.get("fPitchChange", 0))
+        self.spk_id = 1  # 默认说话人。
+        self.spk_mix_dict = None  # {1:0.5, 2:0.5} 表示1号说话人和2号说话人的音色按照0.5:0.5的比例混合
+        self.use_vocoder_based_enhancer = True
+        self.use_phase_vocoder = True
+        self.checkpoint_path = ''
+        self.threhold = -35
+        self.buffer_num = 2
+        self.crossfade_time = 0.03
+        self.select_pitch_extractor = 'harvest'  # F0预测器["parselmouth", "dio", "harvest", "crepe"]
+        self.use_spk_mix = False
+        self.sounddevices = ['', '']
+
+    def save(self, path):
+        with open(path + '\\config.pkl', 'wb') as f:
+            pickle.dump(vars(self), f)
+
+    def load(self, path) -> bool:
+        try:
+            with open(path + '\\config.pkl', 'rb') as f:
+                self.update(pickle.load(f))
+            return True
+        except:
+            print('config.pkl does not exist')
+            return False
+    
+    def update(self, data_dict):
+        for key, value in data_dict.items():
+            setattr(self, key, value)
+
+
+class GUI:
+    def __init__(self) -> None:
+        self.config = Config()
+        self.flag_vc: bool = False  # 变声线程flag
+        self.block_frame = 0
+        self.crossfade_frame = 0
+        self.sola_search_frame = 0
+        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
+        self.svc_model: SvcDDSP = SvcDDSP()
+        self.fade_in_window: np.ndarray = None  # crossfade计算用numpy数组
+        self.fade_out_window: np.ndarray = None  # crossfade计算用numpy数组
+        self.input_wav: np.ndarray = None  # 输入音频规范化后的保存地址
+        self.output_wav: np.ndarray = None  # 输出音频规范化后的保存地址
+        self.sola_buffer: torch.Tensor = None  # 保存上一个output的crossfade
+        self.f0_mode_list = ["parselmouth", "dio", "harvest", "crepe"]  # F0预测器
+        self.f_safe_prefix_pad_length: float = 0.0
+        self.resample_kernel = {}
+        self.launcher()  # start
+
+    def launcher(self):
+        '''窗口加载'''
+        input_devices, output_devices, _, _ = self.get_devices()
+        sg.theme('DarkAmber')  # 设置主题
+        # 界面布局
+        layout = [
+            [sg.Frame(layout=[
+                [sg.Input(key='sg_model', default_text='exp\\multi_speaker\\model_300000.pt'),
+                 sg.FileBrowse(i18n('选择模型文件'), key='choose_model')]
+            ], title=i18n('模型：.pt格式(自动识别同目录下config.yaml)')),
+                sg.Frame(layout=[
+                    [sg.Text(i18n('选择配置文件所在目录')), sg.Input(key='config_file_dir', default_text='exp'),
+                     sg.FolderBrowse(i18n('打开文件夹'), key='choose_config')],
+                    [sg.Button(i18n('读取配置文件'), key='load_config'), sg.Button(i18n('保存配置文件'), key='save_config')]
+                ], title=i18n('快速配置文件'))
+            ],
+            [sg.Frame(layout=[
+                [sg.Text(i18n("输入设备")),
+                 sg.Combo(input_devices, key='sg_input_device', default_value=input_devices[sd.default.device[0]],
+                          enable_events=True)],
+                [sg.Text(i18n("输出设备")),
+                 sg.Combo(output_devices, key='sg_output_device', default_value=output_devices[sd.default.device[1]],
+                          enable_events=True)]
+            ], title=i18n('音频设备'))
+            ],
+            [sg.Frame(layout=[
+                [sg.Text(i18n("说话人id")), sg.Input(key='spk_id', default_text='1')],
+                [sg.Text(i18n("响应阈值")),
+                 sg.Slider(range=(-60, 0), orientation='h', key='threhold', resolution=1, default_value=-45,
+                           enable_events=True)],
+                [sg.Text(i18n("变调")),
+                 sg.Slider(range=(-24, 24), orientation='h', key='pitch', resolution=1, default_value=0,
+                           enable_events=True)],
+                [sg.Text(i18n("采样率")), sg.Input(key='samplerate', default_text='44100')],
+                [sg.Checkbox(text=i18n('启用捏音色功能'), default=False, key='spk_mix', enable_events=True),
+                 sg.Button(i18n("设置混合音色"), key='set_spk_mix')]
+            ], title=i18n('普通设置')),
+                sg.Frame(layout=[
+                    [sg.Text(i18n("音频切分大小")),
+                     sg.Slider(range=(0.05, 3.0), orientation='h', key='block', resolution=0.01, default_value=0.3,
+                               enable_events=True)],
+                    [sg.Text(i18n("交叉淡化时长")),
+                     sg.Slider(range=(0.01, 0.15), orientation='h', key='crossfade', resolution=0.01,
+                               default_value=0.04, enable_events=True)],
+                    [sg.Text(i18n("使用历史区块数量")),
+                     sg.Slider(range=(1, 20), orientation='h', key='buffernum', resolution=1, default_value=4,
+                               enable_events=True)],
+                    [sg.Text(i18n("f0预测模式")),
+                     sg.Combo(values=self.f0_mode_list, key='f0_mode', default_value=self.f0_mode_list[2],
+                              enable_events=True)],
+                    [sg.Checkbox(text=i18n('启用增强器'), default=True, key='use_enhancer', enable_events=True),
+                     sg.Checkbox(text=i18n('启用相位声码器'), default=False, key='use_phase_vocoder', enable_events=True)]
+                ], title=i18n('性能设置')),
+            ],
+            [sg.Button(i18n("开始音频转换"), key="start_vc"), sg.Button(i18n("停止音频转换"), key="stop_vc"),
+             sg.Text(i18n('推理所用时间(ms):')), sg.Text('0', key='infer_time')]
+        ]
+
+        # 创造窗口
+        self.window = sg.Window('DDSP - GUI', layout, finalize=True)
+        self.window['spk_id'].bind('<Return>', '')
+        self.window['samplerate'].bind('<Return>', '')
+        self.event_handler()
+
+    def event_handler(self):
+        '''事件处理'''
+        while True:  # 事件处理循环
+            event, values = self.window.read()
+            print('event: ' + event)
+            if event == sg.WINDOW_CLOSED:  # 如果用户关闭窗口
+                self.flag_vc = False
+                exit()
+            elif event == 'start_vc' and self.flag_vc == False:
+                # set values 和界面布局layout顺序一一对应
+                self.set_values(values)
+                print('crossfade_time:' + str(self.config.crossfade_time))
+                print("buffer_num:" + str(self.config.buffer_num))
+                print("samplerate:" + str(self.config.samplerate))
+                print('block_time:' + str(self.config.block_time))
+                print("prefix_pad_length:" + str(self.f_safe_prefix_pad_length))
+                print("mix_mode:" + str(self.config.spk_mix_dict))
+                print("enhancer:" + str(self.config.use_vocoder_based_enhancer))
+                print('using_cuda:' + str(torch.cuda.is_available()))
+                self.start_vc()
+            elif event == 'spk_id':
+                self.config.spk_id = int(values['spk_id'])
+            elif event == 'threhold':
+                self.config.threhold = values['threhold']
+            elif event == 'pitch':
+                self.config.f_pitch_change = values['pitch']
+            elif event == 'spk_mix':
+                self.config.use_spk_mix = values['spk_mix']
+            elif event == 'set_spk_mix':
+                spk_mix = sg.popup_get_text(message='示例：1:0.3,2:0.5,3:0.2', title="设置混合音色，支持多人")
+                if spk_mix != None:
+                    self.config.spk_mix_dict = eval("{" + spk_mix.replace('，', ',').replace('：', ':') + "}")
+            elif event == 'f0_mode':
+                self.config.select_pitch_extractor = values['f0_mode']
+            elif event == 'use_enhancer':
+                self.config.use_vocoder_based_enhancer = values['use_enhancer']
+            elif event == 'use_phase_vocoder':
+                self.config.use_phase_vocoder = values['use_phase_vocoder']
+            elif event == 'load_config' and self.flag_vc == False:
+                if self.config.load(values['config_file_dir']):
+                    self.update_values()
+            elif event == 'save_config' and self.flag_vc == False:
+                self.set_values(values)
+                self.config.save(values['config_file_dir'])
+            elif event != 'start_vc' and self.flag_vc == True:
+                self.flag_vc = False
+
+    def set_values(self, values):
+        self.set_devices(values["sg_input_device"], values['sg_output_device'])
+        self.config.sounddevices = [values["sg_input_device"], values['sg_output_device']]
+        self.config.checkpoint_path = values['sg_model']
+        self.config.spk_id = int(values['spk_id'])
+        self.config.threhold = values['threhold']
+        self.config.f_pitch_change = values['pitch']
+        self.config.samplerate = int(values['samplerate'])
+        self.config.block_time = float(values['block'])
+        self.config.crossfade_time = float(values['crossfade'])
+        self.config.buffer_num = int(values['buffernum'])
+        self.config.select_pitch_extractor = values['f0_mode']
+        self.config.use_vocoder_based_enhancer = values['use_enhancer']
+        self.config.use_phase_vocoder = values['use_phase_vocoder']
+        self.config.use_spk_mix = values['spk_mix']
+        self.block_frame = int(self.config.block_time * self.config.samplerate)
+        self.crossfade_frame = int(self.config.crossfade_time * self.config.samplerate)
+        self.sola_search_frame = int(0.01 * self.config.samplerate)
+        self.last_delay_frame = int(0.02 * self.config.samplerate)
+        self.input_frames = max(
+            self.block_frame + self.crossfade_frame + self.sola_search_frame + 2 * self.last_delay_frame,
+            (1 + self.config.buffer_num) * self.block_frame)
+        self.f_safe_prefix_pad_length = self.config.block_time * self.config.buffer_num - self.config.crossfade_time - 0.01 - 0.02
+
+    def update_values(self):
+        self.window['sg_model'].update(self.config.checkpoint_path)
+        self.window['sg_input_device'].update(self.config.sounddevices[0])
+        self.window['sg_output_device'].update(self.config.sounddevices[1])
+        self.window['spk_id'].update(self.config.spk_id)
+        self.window['threhold'].update(self.config.threhold)
+        self.window['pitch'].update(self.config.f_pitch_change)
+        self.window['samplerate'].update(self.config.samplerate)
+        self.window['spk_mix'].update(self.config.use_spk_mix)
+        self.window['block'].update(self.config.block_time)
+        self.window['crossfade'].update(self.config.crossfade_time)
+        self.window['buffernum'].update(self.config.buffer_num)
+        self.window['f0_mode'].update(self.config.select_pitch_extractor)
+        self.window['use_enhancer'].update(self.config.use_vocoder_based_enhancer)
+
+    def start_vc(self):
+        '''开始音频转换'''
+        torch.cuda.empty_cache()
+        self.flag_vc = True
+        self.input_wav = np.zeros(self.input_frames, dtype='float32')
+        self.sola_buffer = torch.zeros(self.crossfade_frame, device=self.device)
+        self.fade_in_window = torch.sin(
+            np.pi * torch.arange(0, 1, 1 / self.crossfade_frame, device=self.device) / 2) ** 2
+        self.fade_out_window = 1 - self.fade_in_window
+        self.svc_model.update_model(self.config.checkpoint_path)
+        thread_vc = threading.Thread(target=self.soundinput)
+        thread_vc.start()
+
+    def soundinput(self):
+        '''
+        接受音频输入
+        '''
+        with sd.Stream(callback=self.audio_callback, blocksize=self.block_frame, samplerate=self.config.samplerate,
+                       dtype='float32'):
+            while self.flag_vc:
+                time.sleep(self.config.block_time)
+                print('Audio block passed.')
+        print('ENDing VC')
+
+    def audio_callback(self, indata: np.ndarray, outdata: np.ndarray, frames, times, status):
+        '''
+        音频处理
+        '''
+        start_time = time.perf_counter()
+        print("\nStarting callback")
+        self.input_wav[:] = np.roll(self.input_wav, -self.block_frame)
+        self.input_wav[-self.block_frame:] = librosa.to_mono(indata.T)
+
+        # infer
+        _audio, _model_sr = self.svc_model.infer(
+            self.input_wav,
+            self.config.samplerate,
+            spk_id=self.config.spk_id,
+            threhold=self.config.threhold,
+            pitch_adjust=self.config.f_pitch_change,
+            use_spk_mix=self.config.use_spk_mix,
+            spk_mix_dict=self.config.spk_mix_dict,
+            use_enhancer=self.config.use_vocoder_based_enhancer,
+            pitch_extractor_type=self.config.select_pitch_extractor,
+            safe_prefix_pad_length=self.f_safe_prefix_pad_length,
+        )
+
+        # debug sola
+        '''
+        _audio, _model_sr = self.input_wav, self.config.samplerate
+        rs = int(np.random.uniform(-200,200))
+        print('debug_random_shift: ' + str(rs))
+        _audio = np.roll(_audio, rs)
+        _audio = torch.from_numpy(_audio).to(self.device)
+        '''
+
+        if _model_sr != self.config.samplerate:
+            key_str = str(_model_sr) + '_' + str(self.config.samplerate)
+            if key_str not in self.resample_kernel:
+                self.resample_kernel[key_str] = Resample(_model_sr, self.config.samplerate,
+                                                         lowpass_filter_width=128).to(self.device)
+            _audio = self.resample_kernel[key_str](_audio)
+        temp_wav = _audio[
+                   - self.block_frame - self.crossfade_frame - self.sola_search_frame - self.last_delay_frame: - self.last_delay_frame]
+
+        # sola shift
+        conv_input = temp_wav[None, None, : self.crossfade_frame + self.sola_search_frame]
+        cor_nom = F.conv1d(conv_input, self.sola_buffer[None, None, :])
+        cor_den = torch.sqrt(
+            F.conv1d(conv_input ** 2, torch.ones(1, 1, self.crossfade_frame, device=self.device)) + 1e-8)
+        sola_shift = torch.argmax(cor_nom[0, 0] / cor_den[0, 0])
+        temp_wav = temp_wav[sola_shift: sola_shift + self.block_frame + self.crossfade_frame]
+        print('sola_shift: ' + str(int(sola_shift)))
+
+        # phase vocoder
+        if self.config.use_phase_vocoder:
+            temp_wav[: self.crossfade_frame] = phase_vocoder(
+                self.sola_buffer,
+                temp_wav[: self.crossfade_frame],
+                self.fade_out_window,
+                self.fade_in_window)
+        else:
+            temp_wav[: self.crossfade_frame] *= self.fade_in_window
+            temp_wav[: self.crossfade_frame] += self.sola_buffer * self.fade_out_window
+
+        self.sola_buffer = temp_wav[- self.crossfade_frame:]
+
+        outdata[:] = temp_wav[: - self.crossfade_frame, None].repeat(1, 2).cpu().numpy()
+        end_time = time.perf_counter()
+        print('infer_time: ' + str(end_time - start_time))
+        self.window['infer_time'].update(int((end_time - start_time) * 1000))
+
+    def get_devices(self, update: bool = True):
+        '''获取设备列表'''
+        if update:
+            sd._terminate()
+            sd._initialize()
+        devices = sd.query_devices()
+        hostapis = sd.query_hostapis()
+        for hostapi in hostapis:
+            for device_idx in hostapi["devices"]:
+                devices[device_idx]["hostapi_name"] = hostapi["name"]
+        input_devices = [
+            f"{d['name']} ({d['hostapi_name']})"
+            for d in devices
+            if d["max_input_channels"] > 0
+        ]
+        output_devices = [
+            f"{d['name']} ({d['hostapi_name']})"
+            for d in devices
+            if d["max_output_channels"] > 0
+        ]
+        input_devices_indices = [d["index"] for d in devices if d["max_input_channels"] > 0]
+        output_devices_indices = [
+            d["index"] for d in devices if d["max_output_channels"] > 0
+        ]
+        return input_devices, output_devices, input_devices_indices, output_devices_indices
+
+    def set_devices(self, input_device, output_device):
+        '''设置输出设备'''
+        input_devices, output_devices, input_device_indices, output_device_indices = self.get_devices()
+        sd.default.device[0] = input_device_indices[input_devices.index(input_device)]
+        sd.default.device[1] = output_device_indices[output_devices.index(output_device)]
+        print("input device:" + str(sd.default.device[0]) + ":" + str(input_device))
+        print("output device:" + str(sd.default.device[1]) + ":" + str(output_device))
+
+
+
+if __name__ == "__main__":
+    i18n = gui_locale.I18nAuto()
+    gui = GUI()

DDSP-SVC/gui_diff.py

@@ -0,0 +1,576 @@
+import PySimpleGUI as sg
+import sounddevice as sd
+import torch, librosa, threading, pickle
+from enhancer import Enhancer
+import numpy as np
+from torch.nn import functional as F
+from torchaudio.transforms import Resample
+import torchaudio
+from ddsp.vocoder import load_model, F0_Extractor, Volume_Extractor, Units_Encoder
+from ddsp.core import upsample
+import time
+from gui_diff_locale import I18nAuto
+from diffusion.infer_gt_mel import DiffGtMel
+
+
+def phase_vocoder(a, b, fade_out, fade_in):
+    fa = torch.fft.rfft(a)
+    fb = torch.fft.rfft(b)
+    absab = torch.abs(fa) + torch.abs(fb)
+    n = a.shape[0]
+    if n % 2 == 0:
+        absab[1:-1] *= 2
+    else:
+        absab[1:] *= 2
+    phia = torch.angle(fa)
+    phib = torch.angle(fb)
+    deltaphase = phib - phia
+    deltaphase = deltaphase - 2 * np.pi * torch.floor(deltaphase / 2 / np.pi + 0.5)
+    w = 2 * np.pi * torch.arange(n // 2 + 1).to(a) + deltaphase
+    t = torch.arange(n).unsqueeze(-1).to(a) / n
+    result = a * (fade_out ** 2) + b * (fade_in ** 2) + torch.sum(absab * torch.cos(w * t + phia),
+                                                                  -1) * fade_out * fade_in / n
+    return result
+
+
+class SvcDDSP:
+    def __init__(self) -> None:
+        self.model = None
+        self.units_encoder = None
+        self.encoder_type = None
+        self.encoder_ckpt = None
+        self.enhancer = None
+        self.enhancer_type = None
+        self.enhancer_ckpt = None
+
+    def update_model(self, model_path):
+        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+        # load ddsp model
+        if self.model is None or self.model_path != model_path:
+            self.model, self.args = load_model(model_path, device=self.device)
+            self.model_path = model_path
+
+            # load units encoder
+            if self.units_encoder is None or self.args.data.encoder != self.encoder_type or self.args.data.encoder_ckpt != self.encoder_ckpt:
+                if self.args.data.encoder == 'cnhubertsoftfish':
+                    cnhubertsoft_gate = self.args.data.cnhubertsoft_gate
+                else:
+                    cnhubertsoft_gate = 10
+                self.units_encoder = Units_Encoder(
+                    self.args.data.encoder,
+                    self.args.data.encoder_ckpt,
+                    self.args.data.encoder_sample_rate,
+                    self.args.data.encoder_hop_size,
+                    cnhubertsoft_gate=cnhubertsoft_gate,
+                    device=self.device)
+                self.encoder_type = self.args.data.encoder
+                self.encoder_ckpt = self.args.data.encoder_ckpt
+
+        # load enhancer
+        if self.enhancer is None or self.args.enhancer.type != self.enhancer_type or self.args.enhancer.ckpt != self.enhancer_ckpt:
+            self.enhancer = Enhancer(self.args.enhancer.type, self.args.enhancer.ckpt, device=self.device)
+            self.enhancer_type = self.args.enhancer.type
+            self.enhancer_ckpt = self.args.enhancer.ckpt
+
+    def infer(self,
+              audio,
+              sample_rate,
+              spk_id=1,
+              threhold=-45,
+              pitch_adjust=0,
+              use_spk_mix=False,
+              spk_mix_dict=None,
+              use_enhancer=True,
+              enhancer_adaptive_key='auto',
+              pitch_extractor_type='crepe',
+              f0_min=50,
+              f0_max=1100,
+              safe_prefix_pad_length=0,
+              diff_model=None,
+              diff_acc=None,
+              diff_spk_id=None,
+              diff_use=False,
+              diff_use_dpm=False,
+              k_step=None,
+              diff_silence=False,
+              audio_alignment=False
+              ):
+        print("Infering...")
+        # load input
+        # audio, sample_rate = librosa.load(input_wav, sr=None, mono=True)
+        hop_size = self.args.data.block_size * sample_rate / self.args.data.sampling_rate
+        if audio_alignment:
+            audio_length = len(audio)
+        # safe front silence
+        if safe_prefix_pad_length > 0.03:
+            silence_front = safe_prefix_pad_length - 0.03
+        else:
+            silence_front = 0
+        audio_t = torch.from_numpy(audio).float().unsqueeze(0).to(self.device)
+
+        # extract f0
+        pitch_extractor = F0_Extractor(
+            pitch_extractor_type,
+            sample_rate,
+            hop_size,
+            float(f0_min),
+            float(f0_max))
+        f0 = pitch_extractor.extract(audio, uv_interp=True, device=self.device, silence_front=silence_front)
+        f0 = torch.from_numpy(f0).float().to(self.device).unsqueeze(-1).unsqueeze(0)
+        f0 = f0 * 2 ** (float(pitch_adjust) / 12)
+
+        # extract volume
+        volume_extractor = Volume_Extractor(hop_size)
+        volume = volume_extractor.extract(audio)
+        mask = (volume > 10 ** (float(threhold) / 20)).astype('float')
+        mask = np.pad(mask, (4, 4), constant_values=(mask[0], mask[-1]))
+        mask = np.array([np.max(mask[n: n + 9]) for n in range(len(mask) - 8)])
+        mask = torch.from_numpy(mask).float().to(self.device).unsqueeze(-1).unsqueeze(0)
+        mask = upsample(mask, self.args.data.block_size).squeeze(-1)
+        volume = torch.from_numpy(volume).float().to(self.device).unsqueeze(-1).unsqueeze(0)
+
+        # extract units
+        units = self.units_encoder.encode(audio_t, sample_rate, hop_size)
+
+        # spk_id or spk_mix_dict
+        spk_id = torch.LongTensor(np.array([[spk_id]])).to(self.device)
+        diff_spk_id = torch.LongTensor(np.array([[diff_spk_id]])).to(self.device)
+        dictionary = None
+        if use_spk_mix:
+            dictionary = spk_mix_dict
+
+            # forward and return the output
+        with torch.no_grad():
+            output, _, (s_h, s_n) = self.model(units, f0, volume, spk_id=spk_id, spk_mix_dict=dictionary)
+            if diff_use and diff_model is not None:
+                output = diff_model.infer(output, f0, units, volume, acc=diff_acc, spk_id=diff_spk_id,
+                                          k_step=k_step, use_dpm=diff_use_dpm, silence_front=silence_front, use_silence=diff_silence,
+                                          spk_mix_dict=dictionary)
+            output *= mask
+            if use_enhancer and not diff_use:
+                output, output_sample_rate = self.enhancer.enhance(
+                    output,
+                    self.args.data.sampling_rate,
+                    f0,
+                    self.args.data.block_size,
+                    adaptive_key=enhancer_adaptive_key,
+                    silence_front=silence_front)
+            else:
+                output_sample_rate = self.args.data.sampling_rate
+
+            output = output.squeeze()
+            if audio_alignment:
+                output[:audio_length]
+            return output, output_sample_rate
+
+
+class Config:
+    def __init__(self) -> None:
+        self.samplerate = 44100  # Hz
+        self.block_time = 1.5  # s
+        self.f_pitch_change: float = 0.0  # float(request_form.get("fPitchChange", 0))
+        self.spk_id = 1  # 默认说话人。
+        self.spk_mix_dict = None  # {1:0.5, 2:0.5} 表示1号说话人和2号说话人的音色按照0.5:0.5的比例混合
+        self.use_vocoder_based_enhancer = True
+        self.use_phase_vocoder = True
+        self.checkpoint_path = ''
+        self.threhold = -35
+        self.buffer_num = 2
+        self.crossfade_time = 0.03
+        self.select_pitch_extractor = 'harvest'  # F0预测器["parselmouth", "dio", "harvest", "crepe"]
+        self.use_spk_mix = False
+        self.sounddevices = ['', '']
+        self.diff_use = False
+        self.diff_project = ''
+        self.diff_acc = 10
+        self.diff_spk_id = 0
+        self.k_step = 100
+        self.diff_use_dpm = False
+        self.diff_silence = False
+
+    def save(self, path):
+        with open(path + '\\config.pkl', 'wb') as f:
+            pickle.dump(vars(self), f)
+
+    def load(self, path) -> bool:
+        try:
+            with open(path + '\\config.pkl', 'rb') as f:
+                self.update(pickle.load(f))
+            return True
+        except:
+            print('config.pkl does not exist')
+            return False
+    
+    def update(self, data_dict):
+        for key, value in data_dict.items():
+            setattr(self, key, value)
+
+class GUI:
+    def __init__(self) -> None:
+        self.config = Config()
+        self.flag_vc: bool = False  # 变声线程flag
+        self.block_frame = 0
+        self.crossfade_frame = 0
+        self.sola_search_frame = 0
+        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
+        self.svc_model: SvcDDSP = SvcDDSP()
+        self.diff_model: DiffGtMel = DiffGtMel()
+        self.fade_in_window: np.ndarray = None  # crossfade计算用numpy数组
+        self.fade_out_window: np.ndarray = None  # crossfade计算用numpy数组
+        self.input_wav: np.ndarray = None  # 输入音频规范化后的保存地址
+        self.output_wav: np.ndarray = None  # 输出音频规范化后的保存地址
+        self.sola_buffer: torch.Tensor = None  # 保存上一个output的crossfade
+        self.f0_mode_list = ["parselmouth", "dio", "harvest", "crepe"]  # F0预测器
+        self.f_safe_prefix_pad_length: float = 0.0
+        self.resample_kernel = {}
+        self.launcher()  # start
+
+    def launcher(self):
+        '''窗口加载'''
+        input_devices, output_devices, _, _ = self.get_devices()
+        sg.theme('DarkBlue12')  # 设置主题
+        # 界面布局
+        layout = [
+            [sg.Frame(layout=[
+                [sg.Input(key='sg_model', default_text='exp\\combsub-test\\model_300000.pt'),
+                 sg.FileBrowse(i18n('选择模型文件'), key='choose_model')]
+            ], title=i18n('模型：.pt格式(自动识别同目录下config.yaml)')),
+                sg.Frame(layout=[
+                    [sg.Text(i18n('选择配置文件所在目录')), sg.Input(key='config_file_dir', default_text='exp'),
+                     sg.FolderBrowse(i18n('打开文件夹'), key='choose_config')],
+                    [sg.Button(i18n('读取配置文件'), key='load_config'),
+                     sg.Button(i18n('保存配置文件'), key='save_config')]
+                ], title=i18n('快速配置文件'))
+            ],
+            [sg.Frame(layout=[
+                [sg.Text(i18n("输入设备")),
+                 sg.Combo(input_devices, key='sg_input_device', default_value=input_devices[sd.default.device[0]],
+                          enable_events=True)],
+                [sg.Text(i18n("输出设备")),
+                 sg.Combo(output_devices, key='sg_output_device', default_value=output_devices[sd.default.device[1]],
+                          enable_events=True)]
+            ], title=i18n('音频设备'))
+            ],
+            [sg.Frame(layout=[
+                [sg.Text(i18n("说话人id")), sg.Input(key='spk_id', default_text='1', size=8)],
+                [sg.Text(i18n("响应阈值")),
+                 sg.Slider(range=(-60, 0), orientation='h', key='threhold', resolution=1, default_value=-45,
+                           enable_events=True)],
+                [sg.Text(i18n("变调")),
+                 sg.Slider(range=(-24, 24), orientation='h', key='pitch', resolution=1, default_value=0,
+                           enable_events=True)],
+                [sg.Text(i18n("采样率")), sg.Input(key='samplerate', default_text='44100', size=8)],
+                [sg.Checkbox(text=i18n('启用捏音色功能'), default=False, key='spk_mix', enable_events=True),
+                 sg.Button(i18n("设置混合音色"), key='set_spk_mix')]
+            ], title=i18n('普通设置')),
+                sg.Frame(layout=[
+                    [sg.Text(i18n("音频切分大小")),
+                     sg.Slider(range=(0.05, 3.0), orientation='h', key='block', resolution=0.01, default_value=0.5,
+                               enable_events=True)],
+                    [sg.Text(i18n("交叉淡化时长")),
+                     sg.Slider(range=(0.01, 0.15), orientation='h', key='crossfade', resolution=0.01,
+                               default_value=0.04, enable_events=True)],
+                    [sg.Text(i18n("使用历史区块数量")),
+                     sg.Slider(range=(1, 20), orientation='h', key='buffernum', resolution=1, default_value=3,
+                               enable_events=True)],
+                    [sg.Text(i18n("f0预测模式")),
+                     sg.Combo(values=self.f0_mode_list, key='f0_mode', default_value=self.f0_mode_list[2],
+                              enable_events=True)],
+                    [sg.Checkbox(text=i18n('启用增强器'), default=True, key='use_enhancer', enable_events=True),
+                     sg.Checkbox(text=i18n('启用相位声码器'), default=False, key='use_phase_vocoder',
+                                 enable_events=True)]
+                ], title=i18n('性能设置')),
+                sg.Frame(layout=[
+                    [sg.Text(i18n("扩散模型文件"))],
+                    [sg.Input(key='diff_project', default_text='exp\\diffusion-test\\model_400000.pt'),
+                     sg.FileBrowse(i18n('选择模型文件'), key='choose_model')],
+                    [sg.Text(i18n("扩散说话人id")), sg.Input(key='diff_spk_id', default_text='1', size=18)],
+                    [sg.Text(i18n("扩散深度")), sg.Input(key='k_step', default_text='120', size=18)],
+                    [sg.Text(i18n("扩散加速")), sg.Input(key='diff_acc', default_text='20', size=18)],
+                    [sg.Checkbox(text=i18n('启用DPMs(推荐)'), default=False, key='diff_use_dpm', enable_events=True)],
+                    [sg.Checkbox(text=i18n('启用扩散'), default=True, key='diff_use', enable_events=True),
+                     sg.Checkbox(text=i18n('不扩散安全区(加速但损失效果)'), default=False, key='diff_silence', enable_events=True)]
+                ], title=i18n('扩散设置')),
+            ],
+            [sg.Button(i18n("开始音频转换"), key="start_vc"), sg.Button(i18n("停止音频转换"), key="stop_vc"),
+             sg.Text(i18n('推理所用时间(ms):')), sg.Text('0', key='infer_time')]
+        ]
+
+        # 创造窗口
+        self.window = sg.Window('DDSP - GUI', layout, finalize=True)
+        self.window['spk_id'].bind('<Return>', '')
+        self.window['samplerate'].bind('<Return>', '')
+        self.window['diff_spk_id'].bind('<Return>', '')
+        self.window['k_step'].bind('<Return>', '')
+        self.window['diff_acc'].bind('<Return>', '')
+        self.event_handler()
+
+    def event_handler(self):
+        '''事件处理'''
+        while True:  # 事件处理循环
+            event, values = self.window.read()
+            if event == sg.WINDOW_CLOSED:  # 如果用户关闭窗口
+                self.flag_vc = False
+                exit()
+            
+            print('event: ' + event)
+            
+            if event == 'start_vc' and self.flag_vc == False:
+                # set values 和界面布局layout顺序一一对应
+                self.set_values(values)
+                print('crossfade_time:' + str(self.config.crossfade_time))
+                print("buffer_num:" + str(self.config.buffer_num))
+                print("samplerate:" + str(self.config.samplerate))
+                print('block_time:' + str(self.config.block_time))
+                print("prefix_pad_length:" + str(self.f_safe_prefix_pad_length))
+                print("mix_mode:" + str(self.config.spk_mix_dict))
+                print("enhancer:" + str(self.config.use_vocoder_based_enhancer))
+                print("diffusion:" + str(self.config.diff_use))
+                print('using_cuda:' + str(torch.cuda.is_available()))
+                self.start_vc()
+            elif event == 'k_step':
+                if 1 <= int(values['k_step']) <= 1000:
+                    self.config.k_step = int(values['k_step'])
+                else:
+                    self.window['k_step'].update(1000)
+            elif event == 'diff_acc':
+                if self.config.k_step < int(values['diff_acc']):
+                    self.config.diff_acc = int(self.config.k_step / 4)
+                else:
+                    self.config.diff_acc = int(values['diff_acc'])
+            elif event == 'diff_spk_id':
+                self.config.diff_spk_id = int(values['diff_spk_id'])
+            elif event == 'diff_use':
+                self.config.diff_use = values['diff_use']
+                self.window['use_enhancer'].update(False)
+                self.config.use_vocoder_based_enhancer=False
+            elif event == 'diff_silence':
+                self.config.diff_silence = values['diff_silence']
+            elif event == 'diff_use_dpm':
+                self.config.diff_use_dpm = values['diff_use_dpm']
+            elif event == 'spk_id':
+                self.config.spk_id = int(values['spk_id'])
+            elif event == 'threhold':
+                self.config.threhold = values['threhold']
+            elif event == 'pitch':
+                self.config.f_pitch_change = values['pitch']
+            elif event == 'spk_mix':
+                self.config.use_spk_mix = values['spk_mix']
+            elif event == 'set_spk_mix':
+                spk_mix = sg.popup_get_text(message='示例：1:0.3,2:0.5,3:0.2', title="设置混合音色，支持多人")
+                if spk_mix != None:
+                    self.config.spk_mix_dict = eval("{" + spk_mix.replace('，', ',').replace('：', ':') + "}")
+            elif event == 'f0_mode':
+                self.config.select_pitch_extractor = values['f0_mode']
+            elif event == 'use_enhancer':
+                self.config.use_vocoder_based_enhancer = values['use_enhancer']
+                self.window['diff_use'].update(False)
+                self.config.diff_use = False
+            elif event == 'use_phase_vocoder':
+                self.config.use_phase_vocoder = values['use_phase_vocoder']
+            elif event == 'load_config' and self.flag_vc == False:
+                if self.config.load(values['config_file_dir']):
+                    self.update_values()
+            elif event == 'save_config' and self.flag_vc == False:
+                self.set_values(values)
+                self.config.save(values['config_file_dir'])
+            elif event != 'start_vc' and self.flag_vc == True:
+                self.flag_vc = False
+
+    def set_values(self, values):
+        self.set_devices(values["sg_input_device"], values['sg_output_device'])
+        self.config.sounddevices = [values["sg_input_device"], values['sg_output_device']]
+        self.config.checkpoint_path = values['sg_model']
+        self.config.spk_id = int(values['spk_id'])
+        self.config.threhold = values['threhold']
+        self.config.f_pitch_change = values['pitch']
+        self.config.samplerate = int(values['samplerate'])
+        self.config.block_time = float(values['block'])
+        self.config.crossfade_time = float(values['crossfade'])
+        self.config.buffer_num = int(values['buffernum'])
+        self.config.select_pitch_extractor = values['f0_mode']
+        self.config.use_vocoder_based_enhancer = values['use_enhancer']
+        self.config.use_phase_vocoder = values['use_phase_vocoder']
+        self.config.use_spk_mix = values['spk_mix']
+        self.config.diff_use = values['diff_use']
+        self.config.diff_silence = values['diff_silence']
+        self.config.diff_use_dpm = values['diff_use_dpm']
+        self.config.diff_project = values['diff_project']
+        self.config.diff_acc = int(values['diff_acc'])
+        self.config.diff_spk_id = int(values['diff_spk_id'])
+        self.config.k_step = int(values['k_step'])
+        self.block_frame = int(self.config.block_time * self.config.samplerate)
+        self.crossfade_frame = int(self.config.crossfade_time * self.config.samplerate)
+        self.sola_search_frame = int(0.01 * self.config.samplerate)
+        self.last_delay_frame = int(0.02 * self.config.samplerate)
+        self.input_frames = max(
+            self.block_frame + self.crossfade_frame + self.sola_search_frame + 2 * self.last_delay_frame,
+            (1 + self.config.buffer_num) * self.block_frame)
+        self.f_safe_prefix_pad_length = self.config.block_time * self.config.buffer_num - self.config.crossfade_time - 0.01 - 0.02
+
+    def update_values(self):
+        self.window['sg_model'].update(self.config.checkpoint_path)
+        self.window['sg_input_device'].update(self.config.sounddevices[0])
+        self.window['sg_output_device'].update(self.config.sounddevices[1])
+        self.window['spk_id'].update(self.config.spk_id)
+        self.window['threhold'].update(self.config.threhold)
+        self.window['pitch'].update(self.config.f_pitch_change)
+        self.window['samplerate'].update(self.config.samplerate)
+        self.window['spk_mix'].update(self.config.use_spk_mix)
+        self.window['block'].update(self.config.block_time)
+        self.window['crossfade'].update(self.config.crossfade_time)
+        self.window['buffernum'].update(self.config.buffer_num)
+        self.window['f0_mode'].update(self.config.select_pitch_extractor)
+        self.window['use_enhancer'].update(self.config.use_vocoder_based_enhancer)
+        self.window['diff_use'].update(self.config.diff_use)
+        self.window['diff_silence'].update(self.config.diff_silence)
+        self.window['diff_use_dpm'].update(self.config.diff_use_dpm)
+        self.window['diff_project'].update(self.config.diff_project)
+        self.window['diff_acc'].update(self.config.diff_acc)
+        self.window['diff_spk_id'].update(self.config.diff_spk_id)
+        self.window['k_step'].update(self.config.k_step)
+
+    def start_vc(self):
+        '''开始音频转换'''
+        torch.cuda.empty_cache()
+        self.flag_vc = True
+        self.input_wav = np.zeros(self.input_frames, dtype='float32')
+        self.sola_buffer = torch.zeros(self.crossfade_frame, device=self.device)
+        self.fade_in_window = torch.sin(
+            np.pi * torch.arange(0, 1, 1 / self.crossfade_frame, device=self.device) / 2) ** 2
+        self.fade_out_window = 1 - self.fade_in_window
+        self.svc_model.update_model(self.config.checkpoint_path)
+        if self.config.diff_use:
+            self.diff_model.flush_model(self.config.diff_project, ddsp_config=self.svc_model.args)
+        thread_vc = threading.Thread(target=self.soundinput)
+        thread_vc.start()
+
+    def soundinput(self):
+        '''
+        接受音频输入
+        '''
+        with sd.Stream(callback=self.audio_callback, blocksize=self.block_frame, samplerate=self.config.samplerate,
+                       dtype='float32'):
+            while self.flag_vc:
+                time.sleep(self.config.block_time)
+                print('Audio block passed.')
+        print('ENDing VC')
+
+    def audio_callback(self, indata: np.ndarray, outdata: np.ndarray, frames, times, status):
+        '''
+        音频处理
+        '''
+        start_time = time.perf_counter()
+        print("\nStarting callback")
+        self.input_wav[:] = np.roll(self.input_wav, -self.block_frame)
+        self.input_wav[-self.block_frame:] = librosa.to_mono(indata.T)
+
+        # infer
+        if self.config.diff_use:
+            _diff_model = self.diff_model
+        else:
+            _diff_model = None
+        _audio, _model_sr = self.svc_model.infer(
+            self.input_wav,
+            self.config.samplerate,
+            spk_id=self.config.spk_id,
+            threhold=self.config.threhold,
+            pitch_adjust=self.config.f_pitch_change,
+            use_spk_mix=self.config.use_spk_mix,
+            spk_mix_dict=self.config.spk_mix_dict,
+            use_enhancer=self.config.use_vocoder_based_enhancer,
+            pitch_extractor_type=self.config.select_pitch_extractor,
+            safe_prefix_pad_length=self.f_safe_prefix_pad_length,
+            diff_model=_diff_model,
+            diff_acc=self.config.diff_acc,
+            diff_spk_id=self.config.diff_spk_id,
+            diff_use=self.config.diff_use,
+            diff_use_dpm=self.config.diff_use_dpm,
+            k_step=self.config.k_step,
+            diff_silence=self.config.diff_silence
+        )
+
+        # debug sola
+        '''
+        _audio, _model_sr = self.input_wav, self.config.samplerate
+        rs = int(np.random.uniform(-200,200))
+        print('debug_random_shift: ' + str(rs))
+        _audio = np.roll(_audio, rs)
+        _audio = torch.from_numpy(_audio).to(self.device)
+        '''
+
+        if _model_sr != self.config.samplerate:
+            key_str = str(_model_sr) + '_' + str(self.config.samplerate)
+            if key_str not in self.resample_kernel:
+                self.resample_kernel[key_str] = Resample(_model_sr, self.config.samplerate,
+                                                         lowpass_filter_width=128).to(self.device)
+            _audio = self.resample_kernel[key_str](_audio)
+        temp_wav = _audio[
+                   - self.block_frame - self.crossfade_frame - self.sola_search_frame - self.last_delay_frame: - self.last_delay_frame]
+
+        # sola shift
+        conv_input = temp_wav[None, None, : self.crossfade_frame + self.sola_search_frame]
+        cor_nom = F.conv1d(conv_input, self.sola_buffer[None, None, :])
+        cor_den = torch.sqrt(
+            F.conv1d(conv_input ** 2, torch.ones(1, 1, self.crossfade_frame, device=self.device)) + 1e-8)
+        sola_shift = torch.argmax(cor_nom[0, 0] / cor_den[0, 0])
+        temp_wav = temp_wav[sola_shift: sola_shift + self.block_frame + self.crossfade_frame]
+        print('sola_shift: ' + str(int(sola_shift)))
+
+        # phase vocoder
+        if self.config.use_phase_vocoder:
+            temp_wav[: self.crossfade_frame] = phase_vocoder(
+                self.sola_buffer,
+                temp_wav[: self.crossfade_frame],
+                self.fade_out_window,
+                self.fade_in_window)
+        else:
+            temp_wav[: self.crossfade_frame] *= self.fade_in_window
+            temp_wav[: self.crossfade_frame] += self.sola_buffer * self.fade_out_window
+
+        self.sola_buffer = temp_wav[- self.crossfade_frame:]
+
+        outdata[:] = temp_wav[: - self.crossfade_frame, None].repeat(1, 2).cpu().numpy()
+        end_time = time.perf_counter()
+        print('infer_time: ' + str(end_time - start_time))
+        self.window['infer_time'].update(int((end_time - start_time) * 1000))
+
+    def get_devices(self, update: bool = True):
+        '''获取设备列表'''
+        if update:
+            sd._terminate()
+            sd._initialize()
+        devices = sd.query_devices()
+        hostapis = sd.query_hostapis()
+        for hostapi in hostapis:
+            for device_idx in hostapi["devices"]:
+                devices[device_idx]["hostapi_name"] = hostapi["name"]
+        input_devices = [
+            f"{d['name']} ({d['hostapi_name']})"
+            for d in devices
+            if d["max_input_channels"] > 0
+        ]
+        output_devices = [
+            f"{d['name']} ({d['hostapi_name']})"
+            for d in devices
+            if d["max_output_channels"] > 0
+        ]
+        input_devices_indices = [d["index"] for d in devices if d["max_input_channels"] > 0]
+        output_devices_indices = [
+            d["index"] for d in devices if d["max_output_channels"] > 0
+        ]
+        return input_devices, output_devices, input_devices_indices, output_devices_indices
+
+    def set_devices(self, input_device, output_device):
+        '''设置输出设备'''
+        input_devices, output_devices, input_device_indices, output_device_indices = self.get_devices()
+        sd.default.device[0] = input_device_indices[input_devices.index(input_device)]
+        sd.default.device[1] = output_device_indices[output_devices.index(output_device)]
+        print("input device:" + str(sd.default.device[0]) + ":" + str(input_device))
+        print("output device:" + str(sd.default.device[1]) + ":" + str(output_device))
+
+
+if __name__ == "__main__":
+    i18n = I18nAuto()
+    gui = GUI()

DDSP-SVC/gui_diff_locale.py

@@ -0,0 +1,154 @@
+import locale
+'''
+本地化方式如下所示
+'''
+
+LANGUAGE_LIST = ['zh_CN', 'en_US', 'ja_JP']
+LANGUAGE_ALL = {
+    'zh_CN': {
+        'SUPER': 'END',
+        'LANGUAGE': 'zh_CN',
+        '选择模型文件': '选择模型文件',
+        '模型：.pt格式(自动识别同目录下config.yaml)': '模型：.pt格式(自动识别同目录下config.yaml)',
+        '选择配置文件所在目录': '选择配置文件所在目录',
+        '打开文件夹': '打开文件夹',
+        '读取配置文件': '读取配置文件',
+        '保存配置文件': '保存配置文件',
+        '快速配置文件': '快速配置文件',
+        '输入设备': '输入设备',
+        '输出设备': '输出设备',
+        '音频设备': '音频设备',
+        '说话人id': '说话人id',
+        '响应阈值': '响应阈值',
+        '变调': '变调',
+        '采样率': '采样率',
+        '启用捏音色功能': '启用捏音色功能',
+        '设置混合音色': '设置混合音色',
+        '普通设置': '普通设置',
+        '音频切分大小': '音频切分大小',
+        '交叉淡化时长': '交叉淡化时长',
+        '使用历史区块数量': '使用历史区块数量',
+        'f0预测模式': 'f0预测模式',
+        '启用增强器': '启用增强器',
+        '启用相位声码器': '启用相位声码器',
+        '性能设置': '性能设置',
+        '开始音频转换': '开始音频转换',
+        '停止音频转换': '停止音频转换',
+        '推理所用时间(ms):': '推理所用时间(ms):',
+        '扩散设置': '扩散设置',
+        '启用扩散': '启用扩散',
+        '扩散加速': '扩散加速',
+        '扩散深度': '扩散深度',
+        '扩散说话人id': '扩散说话人id',
+        '扩散模型文件': '扩散模型文件',
+        '不扩散安全区(加速但损失效果)': '不扩散安全区(加速但损失效果)',
+        '启用DPMs(推荐)': '启用DPMs(推荐)'
+    },
+    'en_US': {
+        'SUPER': 'zh_CN',
+        'LANGUAGE': 'en_US',
+        '选择模型文件': 'Select Model File',
+        '模型：.pt格式(自动识别同目录下config.yaml)': 'Model：.pt format(Auto ust config.yaml in here)',
+        '选择配置文件所在目录': 'Select the configuration file directory',
+        '打开文件夹': 'Open folder',
+        '读取配置文件': 'Read config file',
+        '保存配置文件': 'Save config file',
+        '快速配置文件': 'Fast config file',
+        '输入设备': 'Input device',
+        '输出设备': 'Output device',
+        '音频设备': 'Audio devices',
+        '说话人id': 'Speaker ID',
+        '响应阈值': 'Response threshold',
+        '变调': 'Pitch',
+        '采样率': 'Sampling rate',
+        '启用捏音色功能': 'Enable Mix Speaker',
+        '设置混合音色': 'Mix Speaker',
+        '普通设置': 'Normal Settings',
+        '音频切分大小': 'Segmentation size',
+        '交叉淡化时长': 'Cross fade duration',
+        '使用历史区块数量': 'Historical blocks used',
+        'f0预测模式': 'f0Extractor',
+        '启用增强器': 'Enable Enhancer',
+        '启用相位声码器': 'Enable Phase Vocoder',
+        '性能设置': 'Performance settings',
+        '开始音频转换': 'Start conversion',
+        '停止音频转换': 'Stop conversion',
+        '推理所用时间(ms):': 'Inference time(ms):',
+        '扩散设置': '扩散设置',
+        '启用扩散': '启用扩散',
+        '扩散加速': '扩散加速',
+        '扩散深度': '扩散深度',
+        '扩散说话人id': '扩散说话人id',
+        '扩散模型文件': '扩散模型文件',
+        '不扩散安全区(加速但损失效果)': '不扩散安全区(加速但损失效果)',
+        '启用DPMs(推荐)': '启用DPMs(推荐)'
+    },
+    'ja_JP': {
+        'SUPER': 'zh_CN',
+        'LANGUAGE': 'ja_JP',
+        '选择模型文件': 'モデルを選択',
+        '模型：.pt格式(自动识别同目录下config.yaml)': 'モデル：.pt形式（同じディレクトリにあるconfig.yamlを自動認識します）',
+        '选择配置文件所在目录': '設定ファイルを選択',
+        '打开文件夹': 'フォルダを開く',
+        '读取配置文件': '設定ファイルを読み込む',
+        '保存配置文件': '設定ファイルを保存',
+        '快速配置文件': '設定プロファイル',
+        '输入设备': '入力デバイス',
+        '输出设备': '出力デバイス',
+        '音频设备': '音声デバイス',
+        '说话人id': '話者ID',
+        '响应阈值': '応答時の閾値',
+        '变调': '音程',
+        '采样率': 'サンプリングレート',
+        '启用捏音色功能': 'ミキシングを有効化',
+        '设置混合音色': 'ミキシング',
+        '普通设置': '通常設定',
+        '音频切分大小': 'セグメンテーションのサイズ',
+        '交叉淡化���长': 'クロスフェードの間隔',
+        '使用历史区块数量': '使用するヒストリカルブロック数',
+        'f0预测模式': 'f0予測モデル',
+        '启用增强器': 'Enhancerを有効化',
+        '启用相位声码器': 'フェーズボコーダを有効化',
+        '性能设置': 'パフォーマンスの設定',
+        '开始音频转换': '変換開始',
+        '停止音频转换': '変換停止',
+        '推理所用时间(ms):': '推論時間(ms):',
+        '扩散设置': '扩散设置',
+        '启用扩散': '启用扩散',
+        '扩散加速': '扩散加速',
+        '扩散深度': '扩散深度',
+        '扩散说话人id': '扩散说话人id',
+        '扩散模型文件': '扩散模型文件',
+        '不扩散安全区(加速但损失效果)': '不扩散安全区(加速但损失效果)',
+        '启用DPMs(推荐)': '启用DPMs(推荐)'
+    }
+}
+
+
+class I18nAuto:
+    def __init__(self, language=None):
+        self.language_list = LANGUAGE_LIST
+        self.language_all = LANGUAGE_ALL
+        self.language_map = {}
+        if language is None:
+            language = 'auto'
+        if language == 'auto':
+            language = locale.getdefaultlocale()[0]
+            if language not in self.language_list:
+                language = 'zh_CN'
+        self.language = language
+        super_language_list = []
+        while self.language_all[language]['SUPER'] != 'END':
+            super_language_list.append(language)
+            language = self.language_all[language]['SUPER']
+        super_language_list.append('zh_CN')
+        super_language_list.reverse()
+        for _lang in super_language_list:
+            self.read_language(self.language_all[_lang])
+
+    def read_language(self, lang_dict: dict):
+        for _key in lang_dict.keys():
+            self.language_map[_key] = lang_dict[_key]
+
+    def __call__(self, key):
+        return self.language_map[key]

DDSP-SVC/gui_locale.py

@@ -0,0 +1,130 @@
+import locale
+'''
+本地化方式如下所示
+'''
+
+LANGUAGE_LIST = ['zh_CN', 'en_US', 'ja_JP']
+LANGUAGE_ALL = {
+    'zh_CN': {
+        'SUPER': 'END',
+        'LANGUAGE': 'zh_CN',
+        '选择模型文件': '选择模型文件',
+        '模型：.pt格式(自动识别同目录下config.yaml)': '模型：.pt格式(自动识别同目录下config.yaml)',
+        '选择配置文件所在目录': '选择配置文件所在目录',
+        '打开文件夹': '打开文件夹',
+        '读取配置文件': '读取配置文件',
+        '保存配置文件': '保存配置文件',
+        '快速配置文件': '快速配置文件',
+        '输入设备': '输入设备',
+        '输出设备': '输出设备',
+        '音频设备': '音频设备',
+        '说话人id': '说话人id',
+        '响应阈值': '响应阈值',
+        '变调': '变调',
+        '采样率': '采样率',
+        '启用捏音色功能': '启用捏音色功能',
+        '设置混合音色': '设置混合音色',
+        '普通设置': '普通设置',
+        '音频切分大小': '音频切分大小',
+        '交叉淡化时长': '交叉淡化时长',
+        '使用历史区块数量': '使用历史区块数量',
+        'f0预测模式': 'f0预测模式',
+        '启用增强器': '启用增强器',
+        '启用相位声码器': '启用相位声码器',
+        '性能设置': '性能设置',
+        '开始音频转换': '开始音频转换',
+        '停止音频转换': '停止音频转换',
+        '推理所用时间(ms):': '推理所用时间(ms):'
+    },
+    'en_US': {
+        'SUPER': 'zh_CN',
+        'LANGUAGE': 'en_US',
+        '选择模型文件': 'Select Model File',
+        '模型：.pt格式(自动识别同目录下config.yaml)': 'Model：.pt format(Auto ust config.yaml in here)',
+        '选择配置文件所在目录': 'Select the configuration file directory',
+        '打开文件夹': 'Open folder',
+        '读取配置文件': 'Read config file',
+        '保存配置文件': 'Save config file',
+        '快速配置文件': 'Fast config file',
+        '输入设备': 'Input device',
+        '输出设备': 'Output device',
+        '音频设备': 'Audio devices',
+        '说话人id': 'Speaker ID',
+        '响应阈值': 'Response threshold',
+        '变调': 'Pitch',
+        '采样率': 'Sampling rate',
+        '启用捏音色功能': 'Enable Mix Speaker',
+        '设置混合音色': 'Mix Speaker',
+        '普通设置': 'Normal Settings',
+        '音频切分大小': 'Segmentation size',
+        '交叉淡化时长': 'Cross fade duration',
+        '使用历史区块数量': 'Historical blocks used',
+        'f0预测模式': 'f0Extractor',
+        '启用增强器': 'Enable Enhancer',
+        '启用相位声码器': 'Enable Phase Vocoder',
+        '性能设置': 'Performance settings',
+        '开始音频转换': 'Start conversion',
+        '停止音频转换': 'Stop conversion',
+        '推理所用时间(ms):': 'Inference time(ms):'
+    },
+    'ja_JP': {
+        'SUPER': 'zh_CN',
+        'LANGUAGE': 'ja_JP',
+        '选择模型文件': 'モデルを選択',
+        '模型：.pt格式(自动识别同目录下config.yaml)': 'モデル：.pt形式（同じディレクトリにあるconfig.yamlを自動認識します）',
+        '选择配置文件所在目录': '設定ファイルを選択',
+        '打开文件夹': 'フォルダを開く',
+        '读取配置文件': '設定ファイルを読み込む',
+        '保存配置文件': '設定ファイルを保存',
+        '快速配置文件': '設定プロファイル',
+        '输入设备': '入力デバイス',
+        '输出设备': '出力デバイス',
+        '音频设备': '音声デバイス',
+        '说话人id': '話者ID',
+        '响应阈值': '応答時の閾値',
+        '变调': '音程',
+        '采样率': 'サンプリングレート',
+        '启用捏音色功能': 'ミキシングを有効化',
+        '设置混合音色': 'ミキシング',
+        '普通设置': '通常設定',
+        '音频切分大小': 'セグメンテーションのサイズ',
+        '交叉淡化时长': 'クロスフェードの間隔',
+        '使用历史区块数量': '使用するヒストリカルブロック数',
+        'f0预测模式': 'f0予測モデル',
+        '启用增强器': 'Enhancerを有効化',
+        '启用相位声码器': 'フェーズボコーダを有効化',
+        '性能设置': 'パフォーマンスの設定',
+        '开始音频转换': '変換開始',
+        '停止音频转换': '変換停止',
+        '推理所用时间(ms):': '推論時間(ms):'
+    }
+}
+
+
+class I18nAuto:
+    def __init__(self, language=None):
+        self.language_list = LANGUAGE_LIST
+        self.language_all = LANGUAGE_ALL
+        self.language_map = {}
+        if language is None:
+            language = 'auto'
+        if language == 'auto':
+            language = locale.getdefaultlocale()[0]
+            if language not in self.language_list:
+                language = 'zh_CN'
+        self.language = language
+        super_language_list = []
+        while self.language_all[language]['SUPER'] != 'END':
+            super_language_list.append(language)
+            language = self.language_all[language]['SUPER']
+        super_language_list.append('zh_CN')
+        super_language_list.reverse()
+        for _lang in super_language_list:
+            self.read_language(self.language_all[_lang])
+
+    def read_language(self, lang_dict: dict):
+        for _key in lang_dict.keys():
+            self.language_map[_key] = lang_dict[_key]
+
+    def __call__(self, key):
+        return self.language_map[key]

DDSP-SVC/logger/saver.py

@@ -0,0 +1,145 @@
+'''
+author: wayn391@mastertones
+'''
+
+import os
+import json
+import time
+import yaml
+import datetime
+import torch
+import matplotlib.pyplot as plt
+from . import utils
+from torch.utils.tensorboard import SummaryWriter
+
+class Saver(object):
+    def __init__(
+            self, 
+            args,
+            initial_global_step=-1):
+
+        self.expdir = args.env.expdir
+        self.sample_rate = args.data.sampling_rate
+        
+        # cold start
+        self.global_step = initial_global_step
+        self.init_time = time.time()
+        self.last_time = time.time()
+
+        # makedirs
+        os.makedirs(self.expdir, exist_ok=True)       
+
+        # path
+        self.path_log_info = os.path.join(self.expdir, 'log_info.txt')
+
+        # ckpt
+        os.makedirs(self.expdir, exist_ok=True)       
+
+        # writer
+        self.writer = SummaryWriter(os.path.join(self.expdir, 'logs'))
+        
+        # save config
+        path_config = os.path.join(self.expdir, 'config.yaml')
+        with open(path_config, "w") as out_config:
+            yaml.dump(dict(args), out_config)
+
+
+    def log_info(self, msg):
+        '''log method'''
+        if isinstance(msg, dict):
+            msg_list = []
+            for k, v in msg.items():
+                tmp_str = ''
+                if isinstance(v, int):
+                    tmp_str = '{}: {:,}'.format(k, v)
+                else:
+                    tmp_str = '{}: {}'.format(k, v)
+
+                msg_list.append(tmp_str)
+            msg_str = '\n'.join(msg_list)
+        else:
+            msg_str = msg
+        
+        # dsplay
+        print(msg_str)
+
+        # save
+        with open(self.path_log_info, 'a') as fp:
+            fp.write(msg_str+'\n')
+
+    def log_value(self, dict):
+        for k, v in dict.items():
+            self.writer.add_scalar(k, v, self.global_step)
+    
+    def log_spec(self, name, spec, spec_out, vmin=-14, vmax=3.5):  
+        spec_cat = torch.cat([(spec_out - spec).abs() + vmin, spec, spec_out], -1)
+        spec = spec_cat[0]
+        if isinstance(spec, torch.Tensor):
+            spec = spec.cpu().numpy()
+        fig = plt.figure(figsize=(12, 9))
+        plt.pcolor(spec.T, vmin=vmin, vmax=vmax)
+        plt.tight_layout()
+        self.writer.add_figure(name, fig, self.global_step)
+    
+    def log_audio(self, dict):
+        for k, v in dict.items():
+            self.writer.add_audio(k, v, global_step=self.global_step, sample_rate=self.sample_rate)
+    
+    def get_interval_time(self, update=True):
+        cur_time = time.time()
+        time_interval = cur_time - self.last_time
+        if update:
+            self.last_time = cur_time
+        return time_interval
+
+    def get_total_time(self, to_str=True):
+        total_time = time.time() - self.init_time
+        if to_str:
+            total_time = str(datetime.timedelta(
+                seconds=total_time))[:-5]
+        return total_time
+
+    def save_model(
+            self,
+            model, 
+            optimizer,
+            name='model',
+            postfix='',
+            to_json=False):
+        # path
+        if postfix:
+            postfix = '_' + postfix
+        path_pt = os.path.join(
+            self.expdir , name+postfix+'.pt')
+       
+        # check
+        print(' [*] model checkpoint saved: {}'.format(path_pt))
+
+        # save
+        torch.save({
+            'global_step': self.global_step,
+            'model': model.state_dict(),
+            'optimizer': optimizer.state_dict()}, path_pt)
+            
+        # to json
+        if to_json:
+            path_json = os.path.join(
+                self.expdir , name+'.json')
+            utils.to_json(path_params, path_json)
+    
+    def delete_model(self, name='model', postfix=''):
+        # path
+        if postfix:
+            postfix = '_' + postfix
+        path_pt = os.path.join(
+            self.expdir , name+postfix+'.pt')
+       
+        # delete
+        if os.path.exists(path_pt):
+            os.remove(path_pt)
+            print(' [*] model checkpoint deleted: {}'.format(path_pt))
+        
+    def global_step_increment(self):
+        self.global_step += 1
+
+

DDSP-SVC/logger/utils.py

@@ -0,0 +1,122 @@
+import os
+import yaml
+import json
+import pickle
+import torch
+
+def traverse_dir(
+        root_dir,
+        extension,
+        amount=None,
+        str_include=None,
+        str_exclude=None,
+        is_pure=False,
+        is_sort=False,
+        is_ext=True):
+
+    file_list = []
+    cnt = 0
+    for root, _, files in os.walk(root_dir):
+        for file in files:
+            if file.endswith(extension):
+                # path
+                mix_path = os.path.join(root, file)
+                pure_path = mix_path[len(root_dir)+1:] if is_pure else mix_path
+
+                # amount
+                if (amount is not None) and (cnt == amount):
+                    if is_sort:
+                        file_list.sort()
+                    return file_list
+                
+                # check string
+                if (str_include is not None) and (str_include not in pure_path):
+                    continue
+                if (str_exclude is not None) and (str_exclude in pure_path):
+                    continue
+                
+                if not is_ext:
+                    ext = pure_path.split('.')[-1]
+                    pure_path = pure_path[:-(len(ext)+1)]
+                file_list.append(pure_path)
+                cnt += 1
+    if is_sort:
+        file_list.sort()
+    return file_list
+
+
+    
+class DotDict(dict):
+    def __getattr__(*args):         
+        val = dict.get(*args)         
+        return DotDict(val) if type(val) is dict else val   
+
+    __setattr__ = dict.__setitem__    
+    __delattr__ = dict.__delitem__
+
+
+def get_network_paras_amount(model_dict):
+    info = dict()
+    for model_name, model in model_dict.items():
+        # all_params = sum(p.numel() for p in model.parameters())
+        trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
+
+        info[model_name] = trainable_params
+    return info
+
+
+def load_config(path_config):
+    with open(path_config, "r") as config:
+        args = yaml.safe_load(config)
+    args = DotDict(args)
+    # print(args)
+    return args
+
+
+def to_json(path_params, path_json):
+    params = torch.load(path_params, map_location=torch.device('cpu'))
+    raw_state_dict = {}
+    for k, v in params.items():
+        val = v.flatten().numpy().tolist()
+        raw_state_dict[k] = val
+
+    with open(path_json, 'w') as outfile:
+        json.dump(raw_state_dict, outfile,indent= "\t")
+
+
+def convert_tensor_to_numpy(tensor, is_squeeze=True):
+    if is_squeeze:
+        tensor = tensor.squeeze()
+    if tensor.requires_grad:
+        tensor = tensor.detach()
+    if tensor.is_cuda:
+        tensor = tensor.cpu()
+    return tensor.numpy()
+
+           
+def load_model(
+        expdir, 
+        model,
+        optimizer,
+        name='model',
+        postfix='',
+        device='cpu'):
+    if postfix == '':
+        postfix = '_' + postfix
+    path = os.path.join(expdir, name+postfix)
+    path_pt = traverse_dir(expdir, '.pt', is_ext=False)
+    global_step = 0
+    if len(path_pt) > 0:
+        steps = [s[len(path):] for s in path_pt]
+        maxstep = max([int(s) if s.isdigit() else 0 for s in steps])
+        if maxstep >= 0:
+            path_pt = path+str(maxstep)+'.pt'
+        else:
+            path_pt = path+'best.pt'
+        print(' [*] restoring model from', path_pt)
+        ckpt = torch.load(path_pt, map_location=torch.device(device))
+        global_step = ckpt['global_step']
+        model.load_state_dict(ckpt['model'], strict=False)
+        if maxstep != 0:
+            optimizer.load_state_dict(ckpt['optimizer'])
+    return global_step, model, optimizer

DDSP-SVC/main.py

@@ -0,0 +1,282 @@
+import os
+import torch
+import librosa
+import argparse
+import numpy as np
+import soundfile as sf
+import pyworld as pw
+import parselmouth
+import hashlib
+from ast import literal_eval
+from slicer import Slicer
+from ddsp.vocoder import load_model, F0_Extractor, Volume_Extractor, Units_Encoder
+from ddsp.core import upsample
+from enhancer import Enhancer
+from tqdm import tqdm
+
+def parse_args(args=None, namespace=None):
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-m",
+        "--model_path",
+        type=str,
+        required=True,
+        help="path to the model file",
+    )
+    parser.add_argument(
+        "-d",
+        "--device",
+        type=str,
+        default=None,
+        required=False,
+        help="cpu or cuda, auto if not set")
+    parser.add_argument(
+        "-i",
+        "--input",
+        type=str,
+        required=True,
+        help="path to the input audio file",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=str,
+        required=True,
+        help="path to the output audio file",
+    )
+    parser.add_argument(
+        "-id",
+        "--spk_id",
+        type=str,
+        required=False,
+        default=1,
+        help="speaker id (for multi-speaker model) | default: 1",
+    )
+    parser.add_argument(
+        "-mix",
+        "--spk_mix_dict",
+        type=str,
+        required=False,
+        default="None",
+        help="mix-speaker dictionary (for multi-speaker model) | default: None",
+    )
+    parser.add_argument(
+        "-k",
+        "--key",
+        type=str,
+        required=False,
+        default=0,
+        help="key changed (number of semitones) | default: 0",
+    )
+    parser.add_argument(
+        "-e",
+        "--enhance",
+        type=str,
+        required=False,
+        default='true',
+        help="true or false | default: true",
+    )
+    parser.add_argument(
+        "-pe",
+        "--pitch_extractor",
+        type=str,
+        required=False,
+        default='crepe',
+        help="pitch extrator type: parselmouth, dio, harvest, crepe (default)",
+    )
+    parser.add_argument(
+        "-fmin",
+        "--f0_min",
+        type=str,
+        required=False,
+        default=50,
+        help="min f0 (Hz) | default: 50",
+    )
+    parser.add_argument(
+        "-fmax",
+        "--f0_max",
+        type=str,
+        required=False,
+        default=1100,
+        help="max f0 (Hz) | default: 1100",
+    )
+    parser.add_argument(
+        "-th",
+        "--threhold",
+        type=str,
+        required=False,
+        default=-60,
+        help="response threhold (dB) | default: -60",
+    )
+    parser.add_argument(
+        "-eak",
+        "--enhancer_adaptive_key",
+        type=str,
+        required=False,
+        default=0,
+        help="adapt the enhancer to a higher vocal range (number of semitones) | default: 0",
+    )
+    return parser.parse_args(args=args, namespace=namespace)
+
+    
+def split(audio, sample_rate, hop_size, db_thresh = -40, min_len = 5000):
+    slicer = Slicer(
+                sr=sample_rate,
+                threshold=db_thresh,
+                min_length=min_len)       
+    chunks = dict(slicer.slice(audio))
+    result = []
+    for k, v in chunks.items():
+        tag = v["split_time"].split(",")
+        if tag[0] != tag[1]:
+            start_frame = int(int(tag[0]) // hop_size)
+            end_frame = int(int(tag[1]) // hop_size)
+            if end_frame > start_frame:
+                result.append((
+                        start_frame, 
+                        audio[int(start_frame * hop_size) : int(end_frame * hop_size)]))
+    return result
+
+
+def cross_fade(a: np.ndarray, b: np.ndarray, idx: int):
+    result = np.zeros(idx + b.shape[0])
+    fade_len = a.shape[0] - idx
+    np.copyto(dst=result[:idx], src=a[:idx])
+    k = np.linspace(0, 1.0, num=fade_len, endpoint=True)
+    result[idx: a.shape[0]] = (1 - k) * a[idx:] + k * b[: fade_len]
+    np.copyto(dst=result[a.shape[0]:], src=b[fade_len:])
+    return result
+
+
+if __name__ == '__main__':
+    # parse commands
+    cmd = parse_args()
+
+    #device = 'cpu' 
+    device = cmd.device
+    if device is None:
+        device = 'cuda' if torch.cuda.is_available() else 'cpu'
+    
+    # load ddsp model
+    model, args = load_model(cmd.model_path, device=device)
+    
+    # load input
+    audio, sample_rate = librosa.load(cmd.input, sr=None)
+    if len(audio.shape) > 1:
+        audio = librosa.to_mono(audio)
+    hop_size = args.data.block_size * sample_rate / args.data.sampling_rate
+    
+    # get MD5 hash from wav file
+    md5_hash = ""
+    with open(cmd.input, 'rb') as f:
+        data = f.read()
+        md5_hash = hashlib.md5(data).hexdigest()
+        print("MD5: " + md5_hash)
+    
+    cache_dir_path = os.path.join(os.path.dirname(__file__), "cache")
+    cache_file_path = os.path.join(cache_dir_path, f"{cmd.pitch_extractor}_{hop_size}_{cmd.f0_min}_{cmd.f0_max}_{md5_hash}.npy")
+    
+    is_cache_available = os.path.exists(cache_file_path)
+    if is_cache_available:
+        # f0 cache load
+        print('Loading pitch curves for input audio from cache directory...')
+        f0 = np.load(cache_file_path, allow_pickle=False)
+    else:
+        # extract f0
+        print('Pitch extractor type: ' + cmd.pitch_extractor)
+        pitch_extractor = F0_Extractor(
+                            cmd.pitch_extractor, 
+                            sample_rate, 
+                            hop_size, 
+                            float(cmd.f0_min), 
+                            float(cmd.f0_max))
+        print('Extracting the pitch curve of the input audio...')
+        f0 = pitch_extractor.extract(audio, uv_interp = True, device = device)
+        
+        # f0 cache save
+        os.makedirs(cache_dir_path, exist_ok=True)
+        np.save(cache_file_path, f0, allow_pickle=False)
+    
+    f0 = torch.from_numpy(f0).float().to(device).unsqueeze(-1).unsqueeze(0)
+    
+    # key change
+    f0 = f0 * 2 ** (float(cmd.key) / 12)
+    
+    # extract volume 
+    print('Extracting the volume envelope of the input audio...')
+    volume_extractor = Volume_Extractor(hop_size)
+    volume = volume_extractor.extract(audio)
+    mask = (volume > 10 ** (float(cmd.threhold) / 20)).astype('float')
+    mask = np.pad(mask, (4, 4), constant_values=(mask[0], mask[-1]))
+    mask = np.array([np.max(mask[n : n + 9]) for n in range(len(mask) - 8)])
+    mask = torch.from_numpy(mask).float().to(device).unsqueeze(-1).unsqueeze(0)
+    mask = upsample(mask, args.data.block_size).squeeze(-1)
+    volume = torch.from_numpy(volume).float().to(device).unsqueeze(-1).unsqueeze(0)
+    
+    # load units encoder
+    if args.data.encoder == 'cnhubertsoftfish':
+        cnhubertsoft_gate = args.data.cnhubertsoft_gate
+    else:
+        cnhubertsoft_gate = 10
+    units_encoder = Units_Encoder(
+                        args.data.encoder, 
+                        args.data.encoder_ckpt, 
+                        args.data.encoder_sample_rate, 
+                        args.data.encoder_hop_size,
+                        cnhubertsoft_gate=cnhubertsoft_gate,
+                        device = device)
+                        
+    # load enhancer
+    if cmd.enhance == 'true':
+        print('Enhancer type: ' + args.enhancer.type)
+        enhancer = Enhancer(args.enhancer.type, args.enhancer.ckpt, device=device)
+    else:
+        print('Enhancer type: none (using raw output of ddsp)')
+    
+    # speaker id or mix-speaker dictionary
+    spk_mix_dict = literal_eval(cmd.spk_mix_dict)
+    if spk_mix_dict is not None:
+        print('Mix-speaker mode')
+    else:
+        print('Speaker ID: '+ str(int(cmd.spk_id)))        
+    spk_id = torch.LongTensor(np.array([[int(cmd.spk_id)]])).to(device)
+    
+    # forward and save the output
+    result = np.zeros(0)
+    current_length = 0
+    segments = split(audio, sample_rate, hop_size)
+    print('Cut the input audio into ' + str(len(segments)) + ' slices')
+    with torch.no_grad():
+        for segment in tqdm(segments):
+            start_frame = segment[0]
+            seg_input = torch.from_numpy(segment[1]).float().unsqueeze(0).to(device)
+            seg_units = units_encoder.encode(seg_input, sample_rate, hop_size)
+           
+            seg_f0 = f0[:, start_frame : start_frame + seg_units.size(1), :]
+            seg_volume = volume[:, start_frame : start_frame + seg_units.size(1), :]
+            
+            seg_output, _, (s_h, s_n) = model(seg_units, seg_f0, seg_volume, spk_id = spk_id, spk_mix_dict = spk_mix_dict)
+            seg_output *= mask[:, start_frame * args.data.block_size : (start_frame + seg_units.size(1)) * args.data.block_size]
+            
+            if cmd.enhance == 'true':
+                seg_output, output_sample_rate = enhancer.enhance(
+                                                            seg_output, 
+                                                            args.data.sampling_rate, 
+                                                            seg_f0, 
+                                                            args.data.block_size, 
+                                                            adaptive_key = cmd.enhancer_adaptive_key)
+            else:
+                output_sample_rate = args.data.sampling_rate
+            
+            seg_output = seg_output.squeeze().cpu().numpy()
+            
+            silent_length = round(start_frame * args.data.block_size * output_sample_rate / args.data.sampling_rate) - current_length
+            if silent_length >= 0:
+                result = np.append(result, np.zeros(silent_length))
+                result = np.append(result, seg_output)
+            else:
+                result = cross_fade(result, seg_output, current_length + silent_length)
+            current_length = current_length + silent_length + len(seg_output)
+        sf.write(cmd.output, result, output_sample_rate)
+    

DDSP-SVC/main_diff.py

@@ -0,0 +1,372 @@
+import os
+import torch
+import librosa
+import argparse
+import numpy as np
+import soundfile as sf
+import pyworld as pw
+import parselmouth
+import hashlib
+from ast import literal_eval
+from slicer import Slicer
+from ddsp.vocoder import load_model, F0_Extractor, Volume_Extractor, Units_Encoder
+from ddsp.core import upsample
+from diffusion.unit2mel import load_model_vocoder
+from tqdm import tqdm
+
+def check_args(ddsp_args, diff_args):
+    if ddsp_args.data.sampling_rate != diff_args.data.sampling_rate:
+        print("Unmatch data.sampling_rate!")
+        return False
+    if ddsp_args.data.block_size != diff_args.data.block_size:
+        print("Unmatch data.block_size!")
+        return False
+    if ddsp_args.data.encoder != diff_args.data.encoder:
+        print("Unmatch data.encoder!")
+        return False
+    return True
+    
+def parse_args(args=None, namespace=None):
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-diff",
+        "--diff_ckpt",
+        type=str,
+        required=True,
+        help="path to the diffusion model checkpoint",
+    )
+    parser.add_argument(
+        "-ddsp",
+        "--ddsp_ckpt",
+        type=str,
+        required=False,
+        default="None",
+        help="path to the DDSP model checkpoint (for shallow diffusion)",
+    )
+    parser.add_argument(
+        "-d",
+        "--device",
+        type=str,
+        default=None,
+        required=False,
+        help="cpu or cuda, auto if not set")
+    parser.add_argument(
+        "-i",
+        "--input",
+        type=str,
+        required=True,
+        help="path to the input audio file",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=str,
+        required=True,
+        help="path to the output audio file",
+    )
+    parser.add_argument(
+        "-id",
+        "--spk_id",
+        type=str,
+        required=False,
+        default=1,
+        help="speaker id (for multi-speaker model) | default: 1",
+    )
+    parser.add_argument(
+        "-mix",
+        "--spk_mix_dict",
+        type=str,
+        required=False,
+        default="None",
+        help="mix-speaker dictionary (for multi-speaker model) | default: None",
+    )
+    parser.add_argument(
+        "-k",
+        "--key",
+        type=str,
+        required=False,
+        default=0,
+        help="key changed (number of semitones) | default: 0",
+    )
+    parser.add_argument(
+        "-f",
+        "--formant_shift_key",
+        type=str,
+        required=False,
+        default=0,
+        help="formant changed (number of semitones) , only for pitch-augmented model| default: 0",
+    )
+    parser.add_argument(
+        "-pe",
+        "--pitch_extractor",
+        type=str,
+        required=False,
+        default='crepe',
+        help="pitch extrator type: parselmouth, dio, harvest, crepe (default)",
+    )
+    parser.add_argument(
+        "-fmin",
+        "--f0_min",
+        type=str,
+        required=False,
+        default=50,
+        help="min f0 (Hz) | default: 50",
+    )
+    parser.add_argument(
+        "-fmax",
+        "--f0_max",
+        type=str,
+        required=False,
+        default=1100,
+        help="max f0 (Hz) | default: 1100",
+    )
+    parser.add_argument(
+        "-th",
+        "--threhold",
+        type=str,
+        required=False,
+        default=-60,
+        help="response threhold (dB) | default: -60",
+    )
+    parser.add_argument(
+        "-diffid",
+        "--diff_spk_id",
+        type=str,
+        required=False,
+        default='auto',
+        help="diffusion speaker id (for multi-speaker model) | default: auto",
+    )
+    parser.add_argument(
+        "-speedup",
+        "--speedup",
+        type=str,
+        required=False,
+        default='auto',
+        help="speed up | default: auto",
+    )
+    parser.add_argument(
+        "-method",
+        "--method",
+        type=str,
+        required=False,
+        default='auto',
+        help="pndm or dpm-solver | default: auto",
+    )
+    parser.add_argument(
+        "-kstep",
+        "--k_step",
+        type=str,
+        required=False,
+        default=None,
+        help="shallow diffusion steps | default: None",
+    )
+    return parser.parse_args(args=args, namespace=namespace)
+
+    
+def split(audio, sample_rate, hop_size, db_thresh = -40, min_len = 5000):
+    slicer = Slicer(
+                sr=sample_rate,
+                threshold=db_thresh,
+                min_length=min_len)       
+    chunks = dict(slicer.slice(audio))
+    result = []
+    for k, v in chunks.items():
+        tag = v["split_time"].split(",")
+        if tag[0] != tag[1]:
+            start_frame = int(int(tag[0]) // hop_size)
+            end_frame = int(int(tag[1]) // hop_size)
+            if end_frame > start_frame:
+                result.append((
+                        start_frame, 
+                        audio[int(start_frame * hop_size) : int(end_frame * hop_size)]))
+    return result
+
+
+def cross_fade(a: np.ndarray, b: np.ndarray, idx: int):
+    result = np.zeros(idx + b.shape[0])
+    fade_len = a.shape[0] - idx
+    np.copyto(dst=result[:idx], src=a[:idx])
+    k = np.linspace(0, 1.0, num=fade_len, endpoint=True)
+    result[idx: a.shape[0]] = (1 - k) * a[idx:] + k * b[: fade_len]
+    np.copyto(dst=result[a.shape[0]:], src=b[fade_len:])
+    return result
+
+
+if __name__ == '__main__':
+    # parse commands
+    cmd = parse_args()
+    
+    #device = 'cpu' 
+    device = cmd.device
+    if device is None:
+        device = 'cuda' if torch.cuda.is_available() else 'cpu'
+    
+    # load diffusion model
+    model, vocoder, args = load_model_vocoder(cmd.diff_ckpt, device=device)
+    
+    # load input
+    audio, sample_rate = librosa.load(cmd.input, sr=None)
+    if len(audio.shape) > 1:
+        audio = librosa.to_mono(audio)
+    hop_size = args.data.block_size * sample_rate / args.data.sampling_rate
+    
+    # get MD5 hash from wav file
+    md5_hash = ""
+    with open(cmd.input, 'rb') as f:
+        data = f.read()
+        md5_hash = hashlib.md5(data).hexdigest()
+        print("MD5: " + md5_hash)
+    
+    cache_dir_path = os.path.join(os.path.dirname(__file__), "cache")
+    cache_file_path = os.path.join(cache_dir_path, f"{cmd.pitch_extractor}_{hop_size}_{cmd.f0_min}_{cmd.f0_max}_{md5_hash}.npy")
+    
+    is_cache_available = os.path.exists(cache_file_path)
+    if is_cache_available:
+        # f0 cache load
+        print('Loading pitch curves for input audio from cache directory...')
+        f0 = np.load(cache_file_path, allow_pickle=False)
+    else:
+        # extract f0
+        print('Pitch extractor type: ' + cmd.pitch_extractor)
+        pitch_extractor = F0_Extractor(
+                            cmd.pitch_extractor, 
+                            sample_rate, 
+                            hop_size, 
+                            float(cmd.f0_min), 
+                            float(cmd.f0_max))
+        print('Extracting the pitch curve of the input audio...')
+        f0 = pitch_extractor.extract(audio, uv_interp = True, device = device)
+        
+        # f0 cache save
+        os.makedirs(cache_dir_path, exist_ok=True)
+        np.save(cache_file_path, f0, allow_pickle=False)
+    
+    f0 = torch.from_numpy(f0).float().to(device).unsqueeze(-1).unsqueeze(0)
+    
+    # key change
+    f0 = f0 * 2 ** (float(cmd.key) / 12)
+    
+    # formant change
+    formant_shift_key = torch.LongTensor(np.array([[float(cmd.formant_shift_key)]])).to(device)
+    
+    # extract volume 
+    print('Extracting the volume envelope of the input audio...')
+    volume_extractor = Volume_Extractor(hop_size)
+    volume = volume_extractor.extract(audio)
+    mask = (volume > 10 ** (float(cmd.threhold) / 20)).astype('float')
+    mask = np.pad(mask, (4, 4), constant_values=(mask[0], mask[-1]))
+    mask = np.array([np.max(mask[n : n + 9]) for n in range(len(mask) - 8)])
+    mask = torch.from_numpy(mask).float().to(device).unsqueeze(-1).unsqueeze(0)
+    mask = upsample(mask, args.data.block_size).squeeze(-1)
+    volume = torch.from_numpy(volume).float().to(device).unsqueeze(-1).unsqueeze(0)
+    
+    # load units encoder
+    if args.data.encoder == 'cnhubertsoftfish':
+        cnhubertsoft_gate = args.data.cnhubertsoft_gate
+    else:
+        cnhubertsoft_gate = 10
+    units_encoder = Units_Encoder(
+                        args.data.encoder, 
+                        args.data.encoder_ckpt, 
+                        args.data.encoder_sample_rate, 
+                        args.data.encoder_hop_size,
+                        cnhubertsoft_gate=cnhubertsoft_gate,
+                        device = device)
+                            
+    # speaker id or mix-speaker dictionary
+    spk_mix_dict = literal_eval(cmd.spk_mix_dict)
+    spk_id = torch.LongTensor(np.array([[int(cmd.spk_id)]])).to(device)
+    if cmd.diff_spk_id == 'auto':
+        diff_spk_id = spk_id
+    else:
+        diff_spk_id = torch.LongTensor(np.array([[int(cmd.diff_spk_id)]])).to(device)
+    if spk_mix_dict is not None:
+        print('Mix-speaker mode')
+    else:
+        print('DDSP Speaker ID: '+ str(int(cmd.spk_id)))
+        print('Diffusion Speaker ID: '+ str(cmd.diff_spk_id)) 
+    
+    # speed up
+    if cmd.speedup == 'auto':
+        infer_speedup = args.infer.speedup
+    else:
+        infer_speedup = int(cmd.speedup)
+    if cmd.method == 'auto':
+        method = args.infer.method
+    else:
+        method = cmd.method
+    if infer_speedup > 1:
+        print('Sampling method: '+ method)
+        print('Speed up: '+ str(infer_speedup))
+    else:
+        print('Sampling method: DDPM')
+    
+    ddsp = None
+    input_mel = None
+    k_step = None
+    if cmd.k_step is not None:
+        k_step = int(cmd.k_step)
+        print('Shallow diffusion step: ' + str(k_step))
+        if cmd.ddsp_ckpt != "None":
+            # load ddsp model
+            ddsp, ddsp_args = load_model(cmd.ddsp_ckpt, device=device)
+            if not check_args(ddsp_args, args):
+                print("Cannot use this DDSP model for shallow diffusion, gaussian diffusion will be used!")
+                ddsp = None
+        else:
+            print('DDSP model is not identified!')
+            print('Extracting the mel spectrum of the input audio for shallow diffusion...')
+            audio_t = torch.from_numpy(audio).float().unsqueeze(0).to(device)
+            input_mel = vocoder.extract(audio_t, sample_rate)
+            input_mel = torch.cat((input_mel, input_mel[:,-1:,:]), 1)
+    else:
+        print('Shallow diffusion step is not identified, gaussian diffusion will be used!')
+        
+    # forward and save the output
+    result = np.zeros(0)
+    current_length = 0
+    segments = split(audio, sample_rate, hop_size)
+    print('Cut the input audio into ' + str(len(segments)) + ' slices')
+    with torch.no_grad():
+        for segment in tqdm(segments):
+            start_frame = segment[0]
+            seg_input = torch.from_numpy(segment[1]).float().unsqueeze(0).to(device)
+            seg_units = units_encoder.encode(seg_input, sample_rate, hop_size)
+           
+            seg_f0 = f0[:, start_frame : start_frame + seg_units.size(1), :]
+            seg_volume = volume[:, start_frame : start_frame + seg_units.size(1), :]
+            if ddsp is not None:
+                seg_ddsp_f0 = 2 ** (-float(cmd.formant_shift_key) / 12) * seg_f0
+                seg_ddsp_output, _ , (_, _) = ddsp(seg_units, seg_ddsp_f0, seg_volume, spk_id = spk_id, spk_mix_dict = spk_mix_dict)
+                seg_input_mel = vocoder.extract(seg_ddsp_output, args.data.sampling_rate, keyshift=float(cmd.formant_shift_key))
+            elif input_mel != None:
+                seg_input_mel = input_mel[:, start_frame : start_frame + seg_units.size(1), :]
+            else:
+                seg_input_mel = None
+                
+            seg_mel = model(
+                    seg_units, 
+                    seg_f0, 
+                    seg_volume, 
+                    spk_id = diff_spk_id, 
+                    spk_mix_dict = spk_mix_dict,
+                    aug_shift = formant_shift_key,
+                    gt_spec=seg_input_mel,
+                    infer=True, 
+                    infer_speedup=infer_speedup, 
+                    method=method,
+                    k_step=k_step)
+            seg_output = vocoder.infer(seg_mel, seg_f0)
+            seg_output *= mask[:, start_frame * args.data.block_size : (start_frame + seg_units.size(1)) * args.data.block_size]
+            seg_output = seg_output.squeeze().cpu().numpy()
+            
+            silent_length = round(start_frame * args.data.block_size) - current_length
+            if silent_length >= 0:
+                result = np.append(result, np.zeros(silent_length))
+                result = np.append(result, seg_output)
+            else:
+                result = cross_fade(result, seg_output, current_length + silent_length)
+            current_length = current_length + silent_length + len(seg_output)
+        sf.write(cmd.output, result, args.data.sampling_rate)
+    

DDSP-SVC/nsf_hifigan/env.py

@@ -0,0 +1,15 @@
+import os
+import shutil
+
+
+class AttrDict(dict):
+    def __init__(self, *args, **kwargs):
+        super(AttrDict, self).__init__(*args, **kwargs)
+        self.__dict__ = self
+
+
+def build_env(config, config_name, path):
+    t_path = os.path.join(path, config_name)
+    if config != t_path:
+        os.makedirs(path, exist_ok=True)
+        shutil.copyfile(config, os.path.join(path, config_name))

DDSP-SVC/nsf_hifigan/models.py

@@ -0,0 +1,430 @@
+import os
+import json
+from .env import AttrDict
+import numpy as np
+import torch
+import torch.nn.functional as F
+import torch.nn as nn
+from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d
+from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
+from .utils import init_weights, get_padding
+
+LRELU_SLOPE = 0.1
+
+
+def load_model(model_path, device='cuda'):
+    config_file = os.path.join(os.path.split(model_path)[0], 'config.json')
+    with open(config_file) as f:
+        data = f.read()
+
+    json_config = json.loads(data)
+    h = AttrDict(json_config)
+
+    generator = Generator(h).to(device)
+
+    cp_dict = torch.load(model_path, map_location=device)
+    generator.load_state_dict(cp_dict['generator'])
+    generator.eval()
+    generator.remove_weight_norm()
+    del cp_dict
+    return generator, h
+
+
+class ResBlock1(torch.nn.Module):
+    def __init__(self, h, channels, kernel_size=3, dilation=(1, 3, 5)):
+        super(ResBlock1, self).__init__()
+        self.h = h
+        self.convs1 = nn.ModuleList([
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
+                               padding=get_padding(kernel_size, dilation[0]))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
+                               padding=get_padding(kernel_size, dilation[1]))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2],
+                               padding=get_padding(kernel_size, dilation[2])))
+        ])
+        self.convs1.apply(init_weights)
+
+        self.convs2 = nn.ModuleList([
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+                               padding=get_padding(kernel_size, 1))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+                               padding=get_padding(kernel_size, 1))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+                               padding=get_padding(kernel_size, 1)))
+        ])
+        self.convs2.apply(init_weights)
+
+    def forward(self, x):
+        for c1, c2 in zip(self.convs1, self.convs2):
+            xt = F.leaky_relu(x, LRELU_SLOPE)
+            xt = c1(xt)
+            xt = F.leaky_relu(xt, LRELU_SLOPE)
+            xt = c2(xt)
+            x = xt + x
+        return x
+
+    def remove_weight_norm(self):
+        for l in self.convs1:
+            remove_weight_norm(l)
+        for l in self.convs2:
+            remove_weight_norm(l)
+
+
+class ResBlock2(torch.nn.Module):
+    def __init__(self, h, channels, kernel_size=3, dilation=(1, 3)):
+        super(ResBlock2, self).__init__()
+        self.h = h
+        self.convs = nn.ModuleList([
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
+                               padding=get_padding(kernel_size, dilation[0]))),
+            weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
+                               padding=get_padding(kernel_size, dilation[1])))
+        ])
+        self.convs.apply(init_weights)
+
+    def forward(self, x):
+        for c in self.convs:
+            xt = F.leaky_relu(x, LRELU_SLOPE)
+            xt = c(xt)
+            x = xt + x
+        return x
+
+    def remove_weight_norm(self):
+        for l in self.convs:
+            remove_weight_norm(l)
+
+
+class SineGen(torch.nn.Module):
+    """ Definition of sine generator
+    SineGen(samp_rate, harmonic_num = 0,
+            sine_amp = 0.1, noise_std = 0.003,
+            voiced_threshold = 0,
+            flag_for_pulse=False)
+    samp_rate: sampling rate in Hz
+    harmonic_num: number of harmonic overtones (default 0)
+    sine_amp: amplitude of sine-wavefrom (default 0.1)
+    noise_std: std of Gaussian noise (default 0.003)
+    voiced_thoreshold: F0 threshold for U/V classification (default 0)
+    flag_for_pulse: this SinGen is used inside PulseGen (default False)
+    Note: when flag_for_pulse is True, the first time step of a voiced
+        segment is always sin(np.pi) or cos(0)
+    """
+
+    def __init__(self, samp_rate, harmonic_num=0,
+                 sine_amp=0.1, noise_std=0.003,
+                 voiced_threshold=0):
+        super(SineGen, self).__init__()
+        self.sine_amp = sine_amp
+        self.noise_std = noise_std
+        self.harmonic_num = harmonic_num
+        self.dim = self.harmonic_num + 1
+        self.sampling_rate = samp_rate
+        self.voiced_threshold = voiced_threshold
+
+    def _f02uv(self, f0):
+        # generate uv signal
+        uv = torch.ones_like(f0)
+        uv = uv * (f0 > self.voiced_threshold)
+        return uv
+
+    @torch.no_grad()
+    def forward(self, f0, upp):
+        """ sine_tensor, uv = forward(f0)
+        input F0: tensor(batchsize=1, length, dim=1)
+                  f0 for unvoiced steps should be 0
+        output sine_tensor: tensor(batchsize=1, length, dim)
+        output uv: tensor(batchsize=1, length, 1)
+        """
+        f0 = f0.unsqueeze(-1)
+        fn = torch.multiply(f0, torch.arange(1, self.dim + 1, device=f0.device).reshape((1, 1, -1)))
+        rad_values = (fn / self.sampling_rate) % 1  ###%1意味着n_har的乘积无法后处理优化
+        rand_ini = torch.rand(fn.shape[0], fn.shape[2], device=fn.device)
+        rand_ini[:, 0] = 0
+        rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini
+        is_half = rad_values.dtype is not torch.float32
+        tmp_over_one = torch.cumsum(rad_values.double(), 1)  # % 1  #####%1意味着后面的cumsum无法再优化
+        if is_half:
+            tmp_over_one = tmp_over_one.half()
+        else:
+            tmp_over_one = tmp_over_one.float()
+        tmp_over_one *= upp
+        tmp_over_one = F.interpolate(
+            tmp_over_one.transpose(2, 1), scale_factor=upp,
+            mode='linear', align_corners=True
+        ).transpose(2, 1)
+        rad_values = F.interpolate(rad_values.transpose(2, 1), scale_factor=upp, mode='nearest').transpose(2, 1)
+        tmp_over_one %= 1
+        tmp_over_one_idx = (tmp_over_one[:, 1:, :] - tmp_over_one[:, :-1, :]) < 0
+        cumsum_shift = torch.zeros_like(rad_values)
+        cumsum_shift[:, 1:, :] = tmp_over_one_idx * -1.0
+        rad_values = rad_values.double()
+        cumsum_shift = cumsum_shift.double()
+        sine_waves = torch.sin(torch.cumsum(rad_values + cumsum_shift, dim=1) * 2 * np.pi)
+        if is_half:
+            sine_waves = sine_waves.half()
+        else:
+            sine_waves = sine_waves.float()
+        sine_waves = sine_waves * self.sine_amp
+        return sine_waves
+
+
+class SourceModuleHnNSF(torch.nn.Module):
+    """ SourceModule for hn-nsf
+    SourceModule(sampling_rate, harmonic_num=0, sine_amp=0.1,
+                 add_noise_std=0.003, voiced_threshod=0)
+    sampling_rate: sampling_rate in Hz
+    harmonic_num: number of harmonic above F0 (default: 0)
+    sine_amp: amplitude of sine source signal (default: 0.1)
+    add_noise_std: std of additive Gaussian noise (default: 0.003)
+        note that amplitude of noise in unvoiced is decided
+        by sine_amp
+    voiced_threshold: threhold to set U/V given F0 (default: 0)
+    Sine_source, noise_source = SourceModuleHnNSF(F0_sampled)
+    F0_sampled (batchsize, length, 1)
+    Sine_source (batchsize, length, 1)
+    noise_source (batchsize, length 1)
+    uv (batchsize, length, 1)
+    """
+
+    def __init__(self, sampling_rate, harmonic_num=0, sine_amp=0.1,
+                 add_noise_std=0.003, voiced_threshod=0):
+        super(SourceModuleHnNSF, self).__init__()
+
+        self.sine_amp = sine_amp
+        self.noise_std = add_noise_std
+
+        # to produce sine waveforms
+        self.l_sin_gen = SineGen(sampling_rate, harmonic_num,
+                                 sine_amp, add_noise_std, voiced_threshod)
+
+        # to merge source harmonics into a single excitation
+        self.l_linear = torch.nn.Linear(harmonic_num + 1, 1)
+        self.l_tanh = torch.nn.Tanh()
+
+    def forward(self, x, upp):
+        sine_wavs = self.l_sin_gen(x, upp)
+        sine_merge = self.l_tanh(self.l_linear(sine_wavs))
+        return sine_merge
+
+
+class Generator(torch.nn.Module):
+    def __init__(self, h):
+        super(Generator, self).__init__()
+        self.h = h
+        self.num_kernels = len(h.resblock_kernel_sizes)
+        self.num_upsamples = len(h.upsample_rates)
+        self.m_source = SourceModuleHnNSF(
+            sampling_rate=h.sampling_rate,
+            harmonic_num=8
+        )
+        self.noise_convs = nn.ModuleList()
+        self.conv_pre = weight_norm(Conv1d(h.num_mels, h.upsample_initial_channel, 7, 1, padding=3))
+        resblock = ResBlock1 if h.resblock == '1' else ResBlock2
+
+        self.ups = nn.ModuleList()
+        for i, (u, k) in enumerate(zip(h.upsample_rates, h.upsample_kernel_sizes)):
+            c_cur = h.upsample_initial_channel // (2 ** (i + 1))
+            self.ups.append(weight_norm(
+                ConvTranspose1d(h.upsample_initial_channel // (2 ** i), h.upsample_initial_channel // (2 ** (i + 1)),
+                                k, u, padding=(k - u) // 2)))
+            if i + 1 < len(h.upsample_rates):  #
+                stride_f0 = int(np.prod(h.upsample_rates[i + 1:]))
+                self.noise_convs.append(Conv1d(
+                    1, c_cur, kernel_size=stride_f0 * 2, stride=stride_f0, padding=stride_f0 // 2))
+            else:
+                self.noise_convs.append(Conv1d(1, c_cur, kernel_size=1))
+        self.resblocks = nn.ModuleList()
+        ch = h.upsample_initial_channel
+        for i in range(len(self.ups)):
+            ch //= 2
+            for j, (k, d) in enumerate(zip(h.resblock_kernel_sizes, h.resblock_dilation_sizes)):
+                self.resblocks.append(resblock(h, ch, k, d))
+
+        self.conv_post = weight_norm(Conv1d(ch, 1, 7, 1, padding=3))
+        self.ups.apply(init_weights)
+        self.conv_post.apply(init_weights)
+        self.upp = int(np.prod(h.upsample_rates))
+
+    def forward(self, x, f0):
+        har_source = self.m_source(f0, self.upp).transpose(1, 2)
+        x = self.conv_pre(x)
+        for i in range(self.num_upsamples):
+            x = F.leaky_relu(x, LRELU_SLOPE)
+            x = self.ups[i](x)
+            x_source = self.noise_convs[i](har_source)
+            x = x + x_source
+            xs = None
+            for j in range(self.num_kernels):
+                if xs is None:
+                    xs = self.resblocks[i * self.num_kernels + j](x)
+                else:
+                    xs += self.resblocks[i * self.num_kernels + j](x)
+            x = xs / self.num_kernels
+        x = F.leaky_relu(x)
+        x = self.conv_post(x)
+        x = torch.tanh(x)
+
+        return x
+
+    def remove_weight_norm(self):
+        print('Removing weight norm...')
+        for l in self.ups:
+            remove_weight_norm(l)
+        for l in self.resblocks:
+            l.remove_weight_norm()
+        remove_weight_norm(self.conv_pre)
+        remove_weight_norm(self.conv_post)
+
+
+class DiscriminatorP(torch.nn.Module):
+    def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
+        super(DiscriminatorP, self).__init__()
+        self.period = period
+        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
+        self.convs = nn.ModuleList([
+            norm_f(Conv2d(1, 32, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))),
+            norm_f(Conv2d(32, 128, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))),
+            norm_f(Conv2d(128, 512, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))),
+            norm_f(Conv2d(512, 1024, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))),
+            norm_f(Conv2d(1024, 1024, (kernel_size, 1), 1, padding=(2, 0))),
+        ])
+        self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
+
+    def forward(self, x):
+        fmap = []
+
+        # 1d to 2d
+        b, c, t = x.shape
+        if t % self.period != 0:  # pad first
+            n_pad = self.period - (t % self.period)
+            x = F.pad(x, (0, n_pad), "reflect")
+            t = t + n_pad
+        x = x.view(b, c, t // self.period, self.period)
+
+        for l in self.convs:
+            x = l(x)
+            x = F.leaky_relu(x, LRELU_SLOPE)
+            fmap.append(x)
+        x = self.conv_post(x)
+        fmap.append(x)
+        x = torch.flatten(x, 1, -1)
+
+        return x, fmap
+
+
+class MultiPeriodDiscriminator(torch.nn.Module):
+    def __init__(self, periods=None):
+        super(MultiPeriodDiscriminator, self).__init__()
+        self.periods = periods if periods is not None else [2, 3, 5, 7, 11]
+        self.discriminators = nn.ModuleList()
+        for period in self.periods:
+            self.discriminators.append(DiscriminatorP(period))
+
+    def forward(self, y, y_hat):
+        y_d_rs = []
+        y_d_gs = []
+        fmap_rs = []
+        fmap_gs = []
+        for i, d in enumerate(self.discriminators):
+            y_d_r, fmap_r = d(y)
+            y_d_g, fmap_g = d(y_hat)
+            y_d_rs.append(y_d_r)
+            fmap_rs.append(fmap_r)
+            y_d_gs.append(y_d_g)
+            fmap_gs.append(fmap_g)
+
+        return y_d_rs, y_d_gs, fmap_rs, fmap_gs
+
+
+class DiscriminatorS(torch.nn.Module):
+    def __init__(self, use_spectral_norm=False):
+        super(DiscriminatorS, self).__init__()
+        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
+        self.convs = nn.ModuleList([
+            norm_f(Conv1d(1, 128, 15, 1, padding=7)),
+            norm_f(Conv1d(128, 128, 41, 2, groups=4, padding=20)),
+            norm_f(Conv1d(128, 256, 41, 2, groups=16, padding=20)),
+            norm_f(Conv1d(256, 512, 41, 4, groups=16, padding=20)),
+            norm_f(Conv1d(512, 1024, 41, 4, groups=16, padding=20)),
+            norm_f(Conv1d(1024, 1024, 41, 1, groups=16, padding=20)),
+            norm_f(Conv1d(1024, 1024, 5, 1, padding=2)),
+        ])
+        self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1))
+
+    def forward(self, x):
+        fmap = []
+        for l in self.convs:
+            x = l(x)
+            x = F.leaky_relu(x, LRELU_SLOPE)
+            fmap.append(x)
+        x = self.conv_post(x)
+        fmap.append(x)
+        x = torch.flatten(x, 1, -1)
+
+        return x, fmap
+
+
+class MultiScaleDiscriminator(torch.nn.Module):
+    def __init__(self):
+        super(MultiScaleDiscriminator, self).__init__()
+        self.discriminators = nn.ModuleList([
+            DiscriminatorS(use_spectral_norm=True),
+            DiscriminatorS(),
+            DiscriminatorS(),
+        ])
+        self.meanpools = nn.ModuleList([
+            AvgPool1d(4, 2, padding=2),
+            AvgPool1d(4, 2, padding=2)
+        ])
+
+    def forward(self, y, y_hat):
+        y_d_rs = []
+        y_d_gs = []
+        fmap_rs = []
+        fmap_gs = []
+        for i, d in enumerate(self.discriminators):
+            if i != 0:
+                y = self.meanpools[i - 1](y)
+                y_hat = self.meanpools[i - 1](y_hat)
+            y_d_r, fmap_r = d(y)
+            y_d_g, fmap_g = d(y_hat)
+            y_d_rs.append(y_d_r)
+            fmap_rs.append(fmap_r)
+            y_d_gs.append(y_d_g)
+            fmap_gs.append(fmap_g)
+
+        return y_d_rs, y_d_gs, fmap_rs, fmap_gs
+
+
+def feature_loss(fmap_r, fmap_g):
+    loss = 0
+    for dr, dg in zip(fmap_r, fmap_g):
+        for rl, gl in zip(dr, dg):
+            loss += torch.mean(torch.abs(rl - gl))
+
+    return loss * 2
+
+
+def discriminator_loss(disc_real_outputs, disc_generated_outputs):
+    loss = 0
+    r_losses = []
+    g_losses = []
+    for dr, dg in zip(disc_real_outputs, disc_generated_outputs):
+        r_loss = torch.mean((1 - dr) ** 2)
+        g_loss = torch.mean(dg ** 2)
+        loss += (r_loss + g_loss)
+        r_losses.append(r_loss.item())
+        g_losses.append(g_loss.item())
+
+    return loss, r_losses, g_losses
+
+
+def generator_loss(disc_outputs):
+    loss = 0
+    gen_losses = []
+    for dg in disc_outputs:
+        l = torch.mean((1 - dg) ** 2)
+        gen_losses.append(l)
+        loss += l
+
+    return loss, gen_losses

DDSP-SVC/nsf_hifigan/nvSTFT.py

@@ -0,0 +1,129 @@
+import math
+import os
+os.environ["LRU_CACHE_CAPACITY"] = "3"
+import random
+import torch
+import torch.utils.data
+import numpy as np
+import librosa
+from librosa.util import normalize
+from librosa.filters import mel as librosa_mel_fn
+from scipy.io.wavfile import read
+import soundfile as sf
+import torch.nn.functional as F
+
+def load_wav_to_torch(full_path, target_sr=None, return_empty_on_exception=False):
+    sampling_rate = None
+    try:
+        data, sampling_rate = sf.read(full_path, always_2d=True)# than soundfile.
+    except Exception as ex:
+        print(f"'{full_path}' failed to load.\nException:")
+        print(ex)
+        if return_empty_on_exception:
+            return [], sampling_rate or target_sr or 48000
+        else:
+            raise Exception(ex)
+    
+    if len(data.shape) > 1:
+        data = data[:, 0]
+        assert len(data) > 2# check duration of audio file is > 2 samples (because otherwise the slice operation was on the wrong dimension)
+    
+    if np.issubdtype(data.dtype, np.integer): # if audio data is type int
+        max_mag = -np.iinfo(data.dtype).min # maximum magnitude = min possible value of intXX
+    else: # if audio data is type fp32
+        max_mag = max(np.amax(data), -np.amin(data))
+        max_mag = (2**31)+1 if max_mag > (2**15) else ((2**15)+1 if max_mag > 1.01 else 1.0) # data should be either 16-bit INT, 32-bit INT or [-1 to 1] float32
+    
+    data = torch.FloatTensor(data.astype(np.float32))/max_mag
+    
+    if (torch.isinf(data) | torch.isnan(data)).any() and return_empty_on_exception:# resample will crash with inf/NaN inputs. return_empty_on_exception will return empty arr instead of except
+        return [], sampling_rate or target_sr or 48000
+    if target_sr is not None and sampling_rate != target_sr:
+        data = torch.from_numpy(librosa.core.resample(data.numpy(), orig_sr=sampling_rate, target_sr=target_sr))
+        sampling_rate = target_sr
+    
+    return data, sampling_rate
+
+def dynamic_range_compression(x, C=1, clip_val=1e-5):
+    return np.log(np.clip(x, a_min=clip_val, a_max=None) * C)
+
+def dynamic_range_decompression(x, C=1):
+    return np.exp(x) / C
+
+def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
+    return torch.log(torch.clamp(x, min=clip_val) * C)
+
+def dynamic_range_decompression_torch(x, C=1):
+    return torch.exp(x) / C
+
+class STFT():
+    def __init__(self, sr=22050, n_mels=80, n_fft=1024, win_size=1024, hop_length=256, fmin=20, fmax=11025, clip_val=1e-5):
+        self.target_sr = sr
+        
+        self.n_mels     = n_mels
+        self.n_fft      = n_fft
+        self.win_size   = win_size
+        self.hop_length = hop_length
+        self.fmin     = fmin
+        self.fmax     = fmax
+        self.clip_val = clip_val
+        self.mel_basis = {}
+        self.hann_window = {}
+    
+    def get_mel(self, y, keyshift=0, speed=1, center=False):
+        sampling_rate = self.target_sr
+        n_mels     = self.n_mels
+        n_fft      = self.n_fft
+        win_size   = self.win_size
+        hop_length = self.hop_length
+        fmin       = self.fmin
+        fmax       = self.fmax
+        clip_val   = self.clip_val
+        
+        factor = 2 ** (keyshift / 12)       
+        n_fft_new = int(np.round(n_fft * factor))
+        win_size_new = int(np.round(win_size * factor))
+        hop_length_new = int(np.round(hop_length * speed))
+        
+        if torch.min(y) < -1.:
+            print('min value is ', torch.min(y))
+        if torch.max(y) > 1.:
+            print('max value is ', torch.max(y))
+        
+        mel_basis_key = str(fmax)+'_'+str(y.device)
+        if mel_basis_key not in self.mel_basis:
+            mel = librosa_mel_fn(sr=sampling_rate, n_fft=n_fft, n_mels=n_mels, fmin=fmin, fmax=fmax)
+            self.mel_basis[mel_basis_key] = torch.from_numpy(mel).float().to(y.device)
+        
+        keyshift_key = str(keyshift)+'_'+str(y.device)
+        if keyshift_key not in self.hann_window:
+            self.hann_window[keyshift_key] = torch.hann_window(win_size_new).to(y.device)
+        
+        pad_left = (win_size_new - hop_length_new) //2
+        pad_right = max((win_size_new- hop_length_new + 1) //2, win_size_new - y.size(-1) - pad_left)
+        if pad_right < y.size(-1):
+            mode = 'reflect'
+        else:
+            mode = 'constant'
+        y = torch.nn.functional.pad(y.unsqueeze(1), (pad_left, pad_right), mode = mode)
+        y = y.squeeze(1)
+        
+        spec = torch.stft(y, n_fft_new, hop_length=hop_length_new, win_length=win_size_new, window=self.hann_window[keyshift_key],
+                          center=center, pad_mode='reflect', normalized=False, onesided=True, return_complex=True)                          
+        spec = torch.sqrt(spec.real.pow(2) + spec.imag.pow(2) + (1e-9))
+        if keyshift != 0:
+            size = n_fft // 2 + 1
+            resize = spec.size(1)
+            if resize < size:
+                spec = F.pad(spec, (0, 0, 0, size-resize))
+            spec = spec[:, :size, :] * win_size / win_size_new   
+        spec = torch.matmul(self.mel_basis[mel_basis_key], spec)
+        spec = dynamic_range_compression_torch(spec, clip_val=clip_val)
+        return spec
+    
+    def __call__(self, audiopath):
+        audio, sr = load_wav_to_torch(audiopath, target_sr=self.target_sr)
+        spect = self.get_mel(audio.unsqueeze(0)).squeeze(0)
+        return spect
+
+stft = STFT()

DDSP-SVC/nsf_hifigan/utils.py

@@ -0,0 +1,68 @@
+import glob
+import os
+import matplotlib
+import torch
+from torch.nn.utils import weight_norm
+matplotlib.use("Agg")
+import matplotlib.pylab as plt
+
+
+def plot_spectrogram(spectrogram):
+    fig, ax = plt.subplots(figsize=(10, 2))
+    im = ax.imshow(spectrogram, aspect="auto", origin="lower",
+                   interpolation='none')
+    plt.colorbar(im, ax=ax)
+
+    fig.canvas.draw()
+    plt.close()
+
+    return fig
+
+
+def init_weights(m, mean=0.0, std=0.01):
+    classname = m.__class__.__name__
+    if classname.find("Conv") != -1:
+        m.weight.data.normal_(mean, std)
+
+
+def apply_weight_norm(m):
+    classname = m.__class__.__name__
+    if classname.find("Conv") != -1:
+        weight_norm(m)
+
+
+def get_padding(kernel_size, dilation=1):
+    return int((kernel_size*dilation - dilation)/2)
+
+
+def load_checkpoint(filepath, device):
+    assert os.path.isfile(filepath)
+    print("Loading '{}'".format(filepath))
+    checkpoint_dict = torch.load(filepath, map_location=device)
+    print("Complete.")
+    return checkpoint_dict
+
+
+def save_checkpoint(filepath, obj):
+    print("Saving checkpoint to {}".format(filepath))
+    torch.save(obj, filepath)
+    print("Complete.")
+
+
+def del_old_checkpoints(cp_dir, prefix, n_models=2):
+    pattern = os.path.join(cp_dir, prefix + '????????')
+    cp_list = glob.glob(pattern) # get checkpoint paths
+    cp_list = sorted(cp_list)# sort by iter
+    if len(cp_list) > n_models: # if more than n_models models are found
+        for cp in cp_list[:-n_models]:# delete the oldest models other than lastest n_models
+            open(cp, 'w').close()# empty file contents
+            os.unlink(cp)# delete file (move to trash when using Colab)
+
+
+def scan_checkpoint(cp_dir, prefix):
+    pattern = os.path.join(cp_dir, prefix + '????????')
+    cp_list = glob.glob(pattern)
+    if len(cp_list) == 0:
+        return None
+    return sorted(cp_list)[-1]
+

DDSP-SVC/preprocess.py

@@ -0,0 +1,197 @@
+import os
+import numpy as np
+import random
+import librosa
+import torch
+import pyworld as pw
+import parselmouth
+import argparse
+import shutil
+from logger import utils
+from tqdm import tqdm
+from ddsp.vocoder import F0_Extractor, Volume_Extractor, Units_Encoder
+from diffusion.vocoder import Vocoder
+from logger.utils import traverse_dir
+import concurrent.futures
+
+def parse_args(args=None, namespace=None):
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-c",
+        "--config",
+        type=str,
+        required=True,
+        help="path to the config file")
+    parser.add_argument(
+        "-d",
+        "--device",
+        type=str,
+        default=None,
+        required=False,
+        help="cpu or cuda, auto if not set")
+    return parser.parse_args(args=args, namespace=namespace)
+    
+def preprocess(path, f0_extractor, volume_extractor, mel_extractor, units_encoder, sample_rate, hop_size, device = 'cuda', use_pitch_aug = False):
+    
+    path_srcdir  = os.path.join(path, 'audio')
+    path_unitsdir  = os.path.join(path, 'units')
+    path_f0dir  = os.path.join(path, 'f0')
+    path_volumedir  = os.path.join(path, 'volume')
+    path_augvoldir  = os.path.join(path, 'aug_vol')
+    path_meldir  = os.path.join(path, 'mel')
+    path_augmeldir  = os.path.join(path, 'aug_mel')
+    path_skipdir = os.path.join(path, 'skip')
+    
+    # list files
+    filelist =  traverse_dir(
+        path_srcdir,
+        extension='wav',
+        is_pure=True,
+        is_sort=True,
+        is_ext=True)
+    
+    # pitch augmentation dictionary
+    pitch_aug_dict = {}
+    
+    # run  
+    def process(file):
+        ext = file.split('.')[-1]
+        binfile = file[:-(len(ext)+1)]+'.npy'
+        path_srcfile = os.path.join(path_srcdir, file)
+        path_unitsfile = os.path.join(path_unitsdir, binfile)
+        path_f0file = os.path.join(path_f0dir, binfile)
+        path_volumefile = os.path.join(path_volumedir, binfile)
+        path_augvolfile = os.path.join(path_augvoldir, binfile)
+        path_melfile = os.path.join(path_meldir, binfile)
+        path_augmelfile = os.path.join(path_augmeldir, binfile)
+        path_skipfile = os.path.join(path_skipdir, file)
+        
+        # load audio
+        audio, _ = librosa.load(path_srcfile, sr=sample_rate)
+        if len(audio.shape) > 1:
+            audio = librosa.to_mono(audio)
+        audio_t = torch.from_numpy(audio).float().to(device)
+        audio_t = audio_t.unsqueeze(0)
+        
+        # extract volume
+        volume = volume_extractor.extract(audio)
+        
+        # extract mel and volume augmentaion
+        if mel_extractor is not None:
+            mel_t = mel_extractor.extract(audio_t, sample_rate)
+            mel = mel_t.squeeze().to('cpu').numpy()
+            
+            max_amp = float(torch.max(torch.abs(audio_t))) + 1e-5
+            max_shift = min(1, np.log10(1/max_amp))
+            log10_vol_shift = random.uniform(-1, max_shift)
+            if use_pitch_aug:
+                keyshift = random.uniform(-5, 5)
+            else:
+                keyshift = 0
+            
+            aug_mel_t = mel_extractor.extract(audio_t * (10 ** log10_vol_shift), sample_rate, keyshift = keyshift)
+            aug_mel = aug_mel_t.squeeze().to('cpu').numpy()
+            aug_vol = volume_extractor.extract(audio * (10 ** log10_vol_shift))
+            
+        # units encode
+        units_t = units_encoder.encode(audio_t, sample_rate, hop_size)
+        units = units_t.squeeze().to('cpu').numpy()
+        
+        # extract f0
+        f0 = f0_extractor.extract(audio, uv_interp = False)
+        
+        uv = f0 == 0
+        if len(f0[~uv]) > 0:
+            # interpolate the unvoiced f0
+            f0[uv] = np.interp(np.where(uv)[0], np.where(~uv)[0], f0[~uv])
+
+            # save npy     
+            os.makedirs(os.path.dirname(path_unitsfile), exist_ok=True)
+            np.save(path_unitsfile, units)
+            os.makedirs(os.path.dirname(path_f0file), exist_ok=True)
+            np.save(path_f0file, f0)
+            os.makedirs(os.path.dirname(path_volumefile), exist_ok=True)
+            np.save(path_volumefile, volume)
+            if mel_extractor is not None:
+                pitch_aug_dict[file[:-(len(ext)+1)]] = keyshift
+                os.makedirs(os.path.dirname(path_melfile), exist_ok=True)
+                np.save(path_melfile, mel)
+                os.makedirs(os.path.dirname(path_augmelfile), exist_ok=True)
+                np.save(path_augmelfile, aug_mel)
+                os.makedirs(os.path.dirname(path_augvolfile), exist_ok=True)
+                np.save(path_augvolfile, aug_vol)
+        else:
+            print('\n[Error] F0 extraction failed: ' + path_srcfile)
+            os.makedirs(os.path.dirname(path_skipfile), exist_ok=True)
+            shutil.move(path_srcfile, os.path.dirname(path_skipfile))
+            print('This file has been moved to ' + path_skipfile)
+    print('Preprocess the audio clips in :', path_srcdir)
+    
+    # single process
+    for file in tqdm(filelist, total=len(filelist)):
+        process(file)
+    
+    if mel_extractor is not None:
+        path_pitchaugdict = os.path.join(path, 'pitch_aug_dict.npy')
+        np.save(path_pitchaugdict, pitch_aug_dict)
+    # multi-process (have bugs)
+    '''
+    with concurrent.futures.ProcessPoolExecutor(max_workers=2) as executor:
+        list(tqdm(executor.map(process, filelist), total=len(filelist)))
+    '''
+                
+if __name__ == '__main__':
+    # parse commands
+    cmd = parse_args()
+
+    device = cmd.device
+    if device is None:
+        device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+    # load config
+    args = utils.load_config(cmd.config)
+    sample_rate = args.data.sampling_rate
+    hop_size = args.data.block_size
+    
+    # initialize f0 extractor
+    f0_extractor = F0_Extractor(
+                        args.data.f0_extractor, 
+                        args.data.sampling_rate, 
+                        args.data.block_size, 
+                        args.data.f0_min, 
+                        args.data.f0_max)
+    
+    # initialize volume extractor
+    volume_extractor = Volume_Extractor(args.data.block_size)
+    
+    # initialize mel extractor
+    mel_extractor = None
+    use_pitch_aug = False
+    if args.model.type == 'Diffusion':
+        mel_extractor = Vocoder(args.vocoder.type, args.vocoder.ckpt, device = device)
+        if mel_extractor.vocoder_sample_rate != sample_rate or mel_extractor.vocoder_hop_size != hop_size:
+            mel_extractor = None
+            print('Unmatch vocoder parameters, mel extraction is ignored!')
+        elif args.model.use_pitch_aug:
+            use_pitch_aug = True
+    
+    # initialize units encoder
+    if args.data.encoder == 'cnhubertsoftfish':
+        cnhubertsoft_gate = args.data.cnhubertsoft_gate
+    else:
+        cnhubertsoft_gate = 10
+    units_encoder = Units_Encoder(
+                        args.data.encoder, 
+                        args.data.encoder_ckpt, 
+                        args.data.encoder_sample_rate, 
+                        args.data.encoder_hop_size,
+                        cnhubertsoft_gate=cnhubertsoft_gate,
+                        device = device)    
+    
+    # preprocess training set
+    preprocess(args.data.train_path, f0_extractor, volume_extractor, mel_extractor, units_encoder, sample_rate, hop_size, device = device, use_pitch_aug = use_pitch_aug)
+    
+    # preprocess validation set
+    preprocess(args.data.valid_path, f0_extractor, volume_extractor, mel_extractor, units_encoder, sample_rate, hop_size, device = device, use_pitch_aug = False)
+    

DDSP-SVC/pretrain/hubert/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

DDSP-SVC/pretrain/nsf_hifigan/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

DDSP-SVC/requirements.txt

@@ -0,0 +1,25 @@
+einops
+fairseq
+flask
+flask_cors
+gin
+gin_config
+librosa
+local_attention
+matplotlib
+numpy
+praat-parselmouth
+pyworld
+PyYAML
+resampy
+scikit_learn
+scipy
+SoundFile
+tensorboard
+torchcrepe
+tqdm
+transformers
+wave
+pysimplegui
+sounddevice
+gradio

DDSP-SVC/slicer.py

@@ -0,0 +1,146 @@
+import librosa
+import torch
+import torchaudio
+
+
+class Slicer:
+    def __init__(self,
+                 sr: int,
+                 threshold: float = -40.,
+                 min_length: int = 5000,
+                 min_interval: int = 300,
+                 hop_size: int = 20,
+                 max_sil_kept: int = 5000):
+        if not min_length >= min_interval >= hop_size:
+            raise ValueError('The following condition must be satisfied: min_length >= min_interval >= hop_size')
+        if not max_sil_kept >= hop_size:
+            raise ValueError('The following condition must be satisfied: max_sil_kept >= hop_size')
+        min_interval = sr * min_interval / 1000
+        self.threshold = 10 ** (threshold / 20.)
+        self.hop_size = round(sr * hop_size / 1000)
+        self.win_size = min(round(min_interval), 4 * self.hop_size)
+        self.min_length = round(sr * min_length / 1000 / self.hop_size)
+        self.min_interval = round(min_interval / self.hop_size)
+        self.max_sil_kept = round(sr * max_sil_kept / 1000 / self.hop_size)
+
+    def _apply_slice(self, waveform, begin, end):
+        if len(waveform.shape) > 1:
+            return waveform[:, begin * self.hop_size: min(waveform.shape[1], end * self.hop_size)]
+        else:
+            return waveform[begin * self.hop_size: min(waveform.shape[0], end * self.hop_size)]
+
+    # @timeit
+    def slice(self, waveform):
+        if len(waveform.shape) > 1:
+            samples = librosa.to_mono(waveform)
+        else:
+            samples = waveform
+        if samples.shape[0] <= self.min_length:
+            return {"0": {"slice": False, "split_time": f"0,{len(waveform)}"}}
+        rms_list = librosa.feature.rms(y=samples, frame_length=self.win_size, hop_length=self.hop_size).squeeze(0)
+        sil_tags = []
+        silence_start = None
+        clip_start = 0
+        for i, rms in enumerate(rms_list):
+            # Keep looping while frame is silent.
+            if rms < self.threshold:
+                # Record start of silent frames.
+                if silence_start is None:
+                    silence_start = i
+                continue
+            # Keep looping while frame is not silent and silence start has not been recorded.
+            if silence_start is None:
+                continue
+            # Clear recorded silence start if interval is not enough or clip is too short
+            is_leading_silence = silence_start == 0 and i > self.max_sil_kept
+            need_slice_middle = i - silence_start >= self.min_interval and i - clip_start >= self.min_length
+            if not is_leading_silence and not need_slice_middle:
+                silence_start = None
+                continue
+            # Need slicing. Record the range of silent frames to be removed.
+            if i - silence_start <= self.max_sil_kept:
+                pos = rms_list[silence_start: i + 1].argmin() + silence_start
+                if silence_start == 0:
+                    sil_tags.append((0, pos))
+                else:
+                    sil_tags.append((pos, pos))
+                clip_start = pos
+            elif i - silence_start <= self.max_sil_kept * 2:
+                pos = rms_list[i - self.max_sil_kept: silence_start + self.max_sil_kept + 1].argmin()
+                pos += i - self.max_sil_kept
+                pos_l = rms_list[silence_start: silence_start + self.max_sil_kept + 1].argmin() + silence_start
+                pos_r = rms_list[i - self.max_sil_kept: i + 1].argmin() + i - self.max_sil_kept
+                if silence_start == 0:
+                    sil_tags.append((0, pos_r))
+                    clip_start = pos_r
+                else:
+                    sil_tags.append((min(pos_l, pos), max(pos_r, pos)))
+                    clip_start = max(pos_r, pos)
+            else:
+                pos_l = rms_list[silence_start: silence_start + self.max_sil_kept + 1].argmin() + silence_start
+                pos_r = rms_list[i - self.max_sil_kept: i + 1].argmin() + i - self.max_sil_kept
+                if silence_start == 0:
+                    sil_tags.append((0, pos_r))
+                else:
+                    sil_tags.append((pos_l, pos_r))
+                clip_start = pos_r
+            silence_start = None
+        # Deal with trailing silence.
+        total_frames = rms_list.shape[0]
+        if silence_start is not None and total_frames - silence_start >= self.min_interval:
+            silence_end = min(total_frames, silence_start + self.max_sil_kept)
+            pos = rms_list[silence_start: silence_end + 1].argmin() + silence_start
+            sil_tags.append((pos, total_frames + 1))
+        # Apply and return slices.
+        if len(sil_tags) == 0:
+            return {"0": {"slice": False, "split_time": f"0,{len(waveform)}"}}
+        else:
+            chunks = []
+            # 第一段静音并非从头开始，补上有声片段
+            if sil_tags[0][0]:
+                chunks.append(
+                    {"slice": False, "split_time": f"0,{min(waveform.shape[0], sil_tags[0][0] * self.hop_size)}"})
+            for i in range(0, len(sil_tags)):
+                # 标识有声片段（跳过第一段）
+                if i:
+                    chunks.append({"slice": False,
+                                   "split_time": f"{sil_tags[i - 1][1] * self.hop_size},{min(waveform.shape[0], sil_tags[i][0] * self.hop_size)}"})
+                # 标识所有静音片段
+                chunks.append({"slice": True,
+                               "split_time": f"{sil_tags[i][0] * self.hop_size},{min(waveform.shape[0], sil_tags[i][1] * self.hop_size)}"})
+            # 最后一段静音并非结尾，补上结尾片段
+            if sil_tags[-1][1] * self.hop_size < len(waveform):
+                chunks.append({"slice": False, "split_time": f"{sil_tags[-1][1] * self.hop_size},{len(waveform)}"})
+            chunk_dict = {}
+            for i in range(len(chunks)):
+                chunk_dict[str(i)] = chunks[i]
+            return chunk_dict
+
+
+def cut(audio_path, db_thresh=-30, min_len=5000, flask_mode=False, flask_sr=None):
+    if not flask_mode:
+        audio, sr = librosa.load(audio_path, sr=None)
+    else:
+        audio = audio_path
+        sr = flask_sr
+    slicer = Slicer(
+        sr=sr,
+        threshold=db_thresh,
+        min_length=min_len
+    )
+    chunks = slicer.slice(audio)
+    return chunks
+
+
+def chunks2audio(audio_path, chunks):
+    chunks = dict(chunks)
+    audio, sr = torchaudio.load(audio_path)
+    if len(audio.shape) == 2 and audio.shape[1] >= 2:
+        audio = torch.mean(audio, dim=0).unsqueeze(0)
+    audio = audio.cpu().numpy()[0]
+    result = []
+    for k, v in chunks.items():
+        tag = v["split_time"].split(",")
+        if tag[0] != tag[1]:
+            result.append((v["slice"], audio[int(tag[0]):int(tag[1])]))
+    return result, sr

DDSP-SVC/solver.py

@@ -0,0 +1,151 @@
+import os
+import time
+import numpy as np
+import torch
+
+from logger.saver import Saver
+from logger import utils
+
+def test(args, model, loss_func, loader_test, saver):
+    print(' [*] testing...')
+    model.eval()
+
+    # losses
+    test_loss = 0.
+    test_loss_rss = 0.
+    test_loss_uv = 0.
+    
+    # intialization
+    num_batches = len(loader_test)
+    rtf_all = []
+    
+    # run
+    with torch.no_grad():
+        for bidx, data in enumerate(loader_test):
+            fn = data['name'][0]
+            print('--------')
+            print('{}/{} - {}'.format(bidx, num_batches, fn))
+
+            # unpack data
+            for k in data.keys():
+                if k != 'name':
+                    data[k] = data[k].to(args.device)
+            print('>>', data['name'][0])
+
+            # forward
+            st_time = time.time()
+            signal, _, (s_h, s_n) = model(data['units'], data['f0'], data['volume'], data['spk_id'])
+            ed_time = time.time()
+
+            # crop
+            min_len = np.min([signal.shape[1], data['audio'].shape[1]])
+            signal        = signal[:,:min_len]
+            data['audio'] = data['audio'][:,:min_len]
+
+            # RTF
+            run_time = ed_time - st_time
+            song_time = data['audio'].shape[-1] / args.data.sampling_rate
+            rtf = run_time / song_time
+            print('RTF: {}  | {} / {}'.format(rtf, run_time, song_time))
+            rtf_all.append(rtf)
+           
+            # loss
+            loss = loss_func(signal, data['audio'])
+
+            test_loss += loss.item()
+
+            # log
+            saver.log_audio({fn+'/gt.wav': data['audio'], fn+'/pred.wav': signal})
+            
+    # report
+    test_loss /= num_batches
+    
+    # check
+    print(' [test_loss] test_loss:', test_loss)
+    print(' Real Time Factor', np.mean(rtf_all))
+    return test_loss
+
+
+def train(args, initial_global_step, model, optimizer, loss_func, loader_train, loader_test):
+    # saver
+    saver = Saver(args, initial_global_step=initial_global_step)
+
+    # model size
+    params_count = utils.get_network_paras_amount({'model': model})
+    saver.log_info('--- model size ---')
+    saver.log_info(params_count)
+    
+    # run
+    best_loss = np.inf
+    num_batches = len(loader_train)
+    model.train()
+    saver.log_info('======= start training =======')
+    for epoch in range(args.train.epochs):
+        for batch_idx, data in enumerate(loader_train):
+            saver.global_step_increment()
+            optimizer.zero_grad()
+
+            # unpack data
+            for k in data.keys():
+                if k != 'name':
+                    data[k] = data[k].to(args.device)
+            
+            # forward
+            signal, _, (s_h, s_n) = model(data['units'].float(), data['f0'], data['volume'], data['spk_id'], infer=False)
+
+            # loss
+            loss = loss_func(signal, data['audio'])
+            
+            # handle nan loss
+            if torch.isnan(loss):
+                raise ValueError(' [x] nan loss ')
+            else:
+                # backpropagate
+                loss.backward()
+                optimizer.step()
+
+            # log loss
+            if saver.global_step % args.train.interval_log == 0:
+                saver.log_info(
+                    'epoch: {} | {:3d}/{:3d} | {} | batch/s: {:.2f} | loss: {:.3f} | time: {} | step: {}'.format(
+                        epoch,
+                        batch_idx,
+                        num_batches,
+                        args.env.expdir,
+                        args.train.interval_log/saver.get_interval_time(),
+                        loss.item(),
+                        saver.get_total_time(),
+                        saver.global_step
+                    )
+                )
+                
+                saver.log_value({
+                    'train/loss': loss.item()
+                })
+            
+            # validation
+            if saver.global_step % args.train.interval_val == 0:
+                # save latest
+                saver.save_model(model, optimizer, postfix=f'{saver.global_step}')
+
+                # run testing set
+                test_loss = test(args, model, loss_func, loader_test, saver)
+             
+                saver.log_info(
+                    ' --- <validation> --- \nloss: {:.3f}. '.format(
+                        test_loss,
+                    )
+                )
+                
+                saver.log_value({
+                    'validation/loss': test_loss
+                })
+                model.train()
+
+                # save best model
+                if test_loss < best_loss:
+                    saver.log_info(' [V] best model updated.')
+                    saver.save_model(model, optimizer, postfix='best')
+                    best_loss = test_loss
+
+                          

DDSP-SVC/train.py

@@ -0,0 +1,93 @@
+import os
+import argparse
+import torch
+
+from logger import utils
+from data_loaders import get_data_loaders
+from solver import train
+from ddsp.vocoder import Sins, CombSub, CombSubFast
+from ddsp.loss import RSSLoss
+
+
+def parse_args(args=None, namespace=None):
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-c",
+        "--config",
+        type=str,
+        required=True,
+        help="path to the config file")
+    return parser.parse_args(args=args, namespace=namespace)
+
+
+if __name__ == '__main__':
+    # parse commands
+    cmd = parse_args()
+    
+    # load config
+    args = utils.load_config(cmd.config)
+    print(' > config:', cmd.config)
+    print(' >    exp:', args.env.expdir)
+
+    # load model
+    model = None
+    
+    if args.model.type == 'Sins':
+        model = Sins(
+            sampling_rate=args.data.sampling_rate,
+            block_size=args.data.block_size,
+            n_harmonics=args.model.n_harmonics,
+            n_mag_allpass=args.model.n_mag_allpass,
+            n_mag_noise=args.model.n_mag_noise,
+            n_unit=args.data.encoder_out_channels,
+            n_spk=args.model.n_spk)
+ 
+    elif args.model.type == 'CombSub':
+        model = CombSub(
+            sampling_rate=args.data.sampling_rate,
+            block_size=args.data.block_size,
+            n_mag_allpass=args.model.n_mag_allpass,
+            n_mag_harmonic=args.model.n_mag_harmonic,
+            n_mag_noise=args.model.n_mag_noise,
+            n_unit=args.data.encoder_out_channels,
+            n_spk=args.model.n_spk)
+    
+    elif args.model.type == 'CombSubFast':
+        model = CombSubFast(
+            sampling_rate=args.data.sampling_rate,
+            block_size=args.data.block_size,
+            n_unit=args.data.encoder_out_channels,
+            n_spk=args.model.n_spk)
+            
+    else:
+        raise ValueError(f" [x] Unknown Model: {args.model.type}")
+    
+    # load parameters
+    optimizer = torch.optim.AdamW(model.parameters())
+    initial_global_step, model, optimizer = utils.load_model(args.env.expdir, model, optimizer, device=args.device)
+    for param_group in optimizer.param_groups:
+        param_group['lr'] = args.train.lr
+        param_group['weight_decay'] = args.train.weight_decay
+        
+    # loss
+    loss_func = RSSLoss(args.loss.fft_min, args.loss.fft_max, args.loss.n_scale, device = args.device)
+
+    # device
+    if args.device == 'cuda':
+        torch.cuda.set_device(args.env.gpu_id)
+    model.to(args.device)
+    
+    for state in optimizer.state.values():
+        for k, v in state.items():
+            if torch.is_tensor(v):
+                state[k] = v.to(args.device)
+                    
+    loss_func.to(args.device)
+
+    # datas
+    loader_train, loader_valid = get_data_loaders(args, whole_audio=False)
+    
+    # run
+    train(args, initial_global_step, model, optimizer, loss_func, loader_train, loader_valid)
+    

DDSP-SVC/train_diff.py

@@ -0,0 +1,70 @@
+import os
+import argparse
+import torch
+from torch.optim import lr_scheduler
+from logger import utils
+from diffusion.data_loaders import get_data_loaders
+from diffusion.solver import train
+from diffusion.unit2mel import Unit2Mel
+from diffusion.vocoder import Vocoder
+
+
+def parse_args(args=None, namespace=None):
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-c",
+        "--config",
+        type=str,
+        required=True,
+        help="path to the config file")
+    return parser.parse_args(args=args, namespace=namespace)
+
+
+if __name__ == '__main__':
+    # parse commands
+    cmd = parse_args()
+    
+    # load config
+    args = utils.load_config(cmd.config)
+    print(' > config:', cmd.config)
+    print(' >    exp:', args.env.expdir)
+    
+    # load vocoder
+    vocoder = Vocoder(args.vocoder.type, args.vocoder.ckpt, device=args.device)
+    
+    # load model
+    model = Unit2Mel(
+                args.data.encoder_out_channels, 
+                args.model.n_spk,
+                args.model.use_pitch_aug,
+                vocoder.dimension,
+                args.model.n_layers,
+                args.model.n_chans,
+                args.model.n_hidden)
+    
+    
+    # load parameters
+    optimizer = torch.optim.AdamW(model.parameters())
+    initial_global_step, model, optimizer = utils.load_model(args.env.expdir, model, optimizer, device=args.device)
+    for param_group in optimizer.param_groups:
+        param_group['lr'] = args.train.lr
+        param_group['weight_decay'] = args.train.weight_decay
+    scheduler = lr_scheduler.StepLR(optimizer, step_size=args.train.decay_step, gamma=args.train.gamma)
+    
+    # device
+    if args.device == 'cuda':
+        torch.cuda.set_device(args.env.gpu_id)
+    model.to(args.device)
+    
+    for state in optimizer.state.values():
+        for k, v in state.items():
+            if torch.is_tensor(v):
+                state[k] = v.to(args.device)
+                    
+    # datas
+    loader_train, loader_valid = get_data_loaders(args, whole_audio=False)
+    
+    # run
+    train(args, initial_global_step, model, optimizer, scheduler, vocoder, loader_train, loader_valid)
+    

DDSP-SVC/webui.py

@@ -0,0 +1,267 @@
+import gradio as gr
+import os,subprocess,yaml
+
+class WebUI:
+    def __init__(self) -> None:
+        self.info=Info()
+        self.opt_cfg_pth='configs/opt.yaml'
+        self.main_ui()
+    
+    def main_ui(self):
+        with gr.Blocks() as ui:
+            gr.Markdown('## 一个便于训练和推理的DDSP-webui，每一步的说明在下面，可以自己展开看。')
+            with gr.Tab("训练/Training"):
+                gr.Markdown(self.info.general)
+                with gr.Accordion('预训练模型说明',open=False):
+                    gr.Markdown(self.info.pretrain_model)
+                with gr.Accordion('数据集说明',open=False):
+                    gr.Markdown(self.info.dataset)
+                
+                gr.Markdown('## 生成配置文件')
+                with gr.Row():
+                    self.batch_size=gr.Slider(minimum=2,maximum=60,value=24,label='Batch_size',interactive=True)
+                    self.learning_rate=gr.Number(value=0.0005,label='学习率',info='和batch_size关系大概是0.0001:6')
+                    self.f0_extractor=gr.Dropdown(['parselmouth', 'dio', 'harvest', 'crepe'],type='value',value='crepe',label='f0提取器种类',interactive=True)
+                    self.sampling_rate=gr.Number(value=44100,label='采样率',info='数据集音频的采样率',interactive=True)
+                    self.n_spk=gr.Number(value=1,label='说话人数量',interactive=True)
+                with gr.Row():
+                    self.device=gr.Dropdown(['cuda','cpu'],value='cuda',label='使用设备',interactive=True)
+                    self.num_workers=gr.Number(value=2,label='读取数据进程数',info='如果你的设备性能很好，可以设置为0',interactive=True)
+                    self.cache_all_data=gr.Checkbox(value=True,label='启用缓存',info='将数据全部加载以加速训练',interactive=True)
+                    self.cache_device=gr.Dropdown(['cuda','cpu'],value='cuda',type='value',label='缓存设备',info='如果你的显存比较大，设置为cuda',interactive=True)
+                self.bt_create_config=gr.Button(value='创建配置文件')
+                
+                gr.Markdown('## 预处理')
+                with gr.Accordion('预训练说明',open=False):
+                    gr.Markdown(self.info.preprocess)
+                with gr.Row():
+                    self.bt_open_data_folder=gr.Button('打开数据集文件夹')
+                    self.bt_preprocess=gr.Button('开始预处理')
+                gr.Markdown('## 训练')
+                with gr.Accordion('训练说明',open=False):
+                    gr.Markdown(self.info.train)
+                with gr.Row():
+                    self.bt_train=gr.Button('开始训练')
+                    self.bt_visual=gr.Button('启动可视化')
+                    gr.Markdown('启动可视化后[点击打开](http://127.0.0.1:6006)')
+                
+            with gr.Tab('推理/Inference'):
+                with gr.Accordion('推理说明',open=False):
+                    gr.Markdown(self.info.infer)
+                with gr.Row():
+                    self.input_wav=gr.Audio(type='filepath',label='选择待转换音频')
+                    self.choose_model=gr.Textbox('exp/model_chino.pt',label='模型路径')
+                with gr.Row():
+                    self.keychange=gr.Slider(-24,24,value=0,step=1,label='变调')
+                    self.id=gr.Number(value=1,label='说话人id')
+                    self.enhancer_adaptive_key=gr.Number(value=0,label='增强器音区偏移',info='调高可以防止超高音（比如大于G5) 破音,但是低音效果可能会下降')
+                with gr.Row():
+                    self.bt_infer=gr.Button(value='开始转换')
+                    self.output_wav=gr.Audio(type='filepath',label='输出音频')
+                    
+            self.bt_create_config.click(fn=self.create_config)
+            self.bt_open_data_folder.click(fn=self.openfolder)
+            self.bt_preprocess.click(fn=self.preprocess)
+            self.bt_train.click(fn=self.training)
+            self.bt_visual.click(fn=self.visualize)
+            self.bt_infer.click(fn=self.inference,inputs=[self.input_wav,self.choose_model,self.keychange,self.id,self.enhancer_adaptive_key],outputs=self.output_wav)
+        ui.launch(inbrowser=True,server_port=7858)
+        
+    def openfolder(self):
+        try:
+            os.startfile('data')
+        except:
+            print('Fail to open folder!')
+
+
+    def create_config(self):
+        with open('configs/combsub.yaml','r',encoding='utf-8') as f:
+            cfg=yaml.load(f.read(),Loader=yaml.FullLoader)
+        cfg['data']['f0_extractor']=str(self.f0_extractor.value)
+        cfg['data']['sampling_rate']=int(self.sampling_rate.value)
+        cfg['train']['batch_size']=int(self.batch_size.value)
+        cfg['device']=str(self.device.value)
+        cfg['train']['num_workers']=int(self.num_workers.value)
+        cfg['train']['cache_all_data']=str(self.cache_all_data.value)
+        cfg['train']['cache_device']=str(self.cache_device.value)
+        cfg['train']['lr']=int(self.learning_rate.value)
+        print('配置文件信息：'+str(cfg))
+        with open(self.opt_cfg_pth,'w',encoding='utf-8') as f:
+            yaml.dump(cfg,f)
+        print('成功生成配置文件')
+
+    
+    def preprocess(self):
+        preprocessing_process=subprocess.Popen('python -u preprocess.py -c '+self.opt_cfg_pth,stdout=subprocess.PIPE)
+        while preprocessing_process.poll() is None:
+            output=preprocessing_process.stdout.readline().decode('utf-8')
+            print(output)
+        print('预处理完成')
+            
+    def training(self):
+        train_process=subprocess.Popen('python -u train.py -c '+self.opt_cfg_pth,stdout=subprocess.PIPE)
+        while train_process.poll() is None:
+            output=train_process.stdout.readline().decode('utf-8')
+            print(output)
+        
+            
+    def visualize(self):
+        tb_process=subprocess.Popen('tensorboard --logdir=exp --port=6006',stdout=subprocess.PIPE)
+        while tb_process.poll() is None:
+            output=tb_process.stdout.readline().decode('utf-8')
+            print(output)
+            
+    def inference(self,input_wav:str,model:str,keychange,id,enhancer_adaptive_key):
+        print(input_wav,model)
+        output_wav='samples/'+ input_wav.replace('\\','/').split('/')[-1]
+        cmd='python -u main.py -i '+input_wav+' -m '+model+' -o '+output_wav+' -k '+str(int(keychange))+' -id '+str(int(id))+' -e true -eak '+str(int(enhancer_adaptive_key))
+        infer_process=subprocess.Popen(cmd,stdout=subprocess.PIPE)
+        while infer_process.poll() is None:
+            output=infer_process.stdout.readline().decode('utf-8')
+            print(output)
+        print('推理完成')
+        return output_wav
+
+
+class Info:
+    def __init__(self) -> None:
+        self.general='''
+### 不看也没事，大致就是  
+1.设置好配置之后点击创建配置文件  
+2.点击‘打开数据集文件夹’，把数据集选个十个塞到data\\train\\val目录下面，剩下的音频全塞到data\\train\\audio下面  
+3.点击‘开始预处理’等待执行完毕  
+4.点击‘开始训练’和‘启动可视化’然后点击右侧链接  
+'''
+        self.pretrain_model="""
+- **(必要操作)** 下载预训练 [**HubertSoft**](https://github.com/bshall/hubert/releases/download/v0.1/hubert-soft-0d54a1f4.pt) 编码器并将其放到 `pretrain/hubert` 文件夹。
+  - 更新：现在支持 ContentVec 编码器了。你可以下载预训练 [ContentVec](https://ibm.ent.box.com/s/z1wgl1stco8ffooyatzdwsqn2psd9lrr) 编码器替代 HubertSoft 编码器并修改配置文件以使用它。
+- 从 [DiffSinger 社区声码器项目](https://openvpi.github.io/vocoders) 下载基于预训练声码器的增强器，并解压至 `pretrain/` 文件夹。
+  -  注意：你应当下载名称中带有`nsf_hifigan`的压缩文件，而非`nsf_hifigan_finetune`。
+        """
+        self.dataset="""
+### 1. 配置训练数据集和验证数据集
+
+#### 1.1 手动配置：
+
+将所有的训练集数据 (.wav 格式音频切片) 放到 `data/train/audio`。
+
+将所有的验证集数据 (.wav 格式音频切片) 放到 `data/val/audio`。
+
+#### 1.2 程序随机选择（**多人物时不可使用**）：
+
+运行`python draw.py`,程序将帮助你挑选验证集数据（可以调整 `draw.py` 中的参数修改抽取文件的数量等参数）。
+
+#### 1.3文件夹结构目录展示：
+- 单人物目录结构：
+
+```
+data
+├─ train
+│    ├─ audio
+│    │    ├─ aaa.wav
+│    │    ├─ bbb.wav
+│    │    └─ ....wav
+│    └─ val
+│    │    ├─ eee.wav
+│    │    ├─ fff.wav
+│    │    └─ ....wav
+ ```
+- 多人物目录结构：
+
+```
+data
+├─ train
+│    ├─ audio
+│    │    ├─ 1
+│    │    │   ├─ aaa.wav
+│    │    │   ├─ bbb.wav
+│    │    │   └─ ....wav
+│    │    ├─ 2
+│    │    │   ├─ ccc.wav
+│    │    │   ├─ ddd.wav
+│    │    │   └─ ....wav
+│    │    └─ ...
+│    └─ val
+│    │    ├─ 1
+│    │    │   ├─ eee.wav
+│    │    │   ├─ fff.wav
+│    │    │   └─ ....wav
+│    │    ├─ 2
+│    │    │   ├─ ggg.wav
+│    │    │   ├─ hhh.wav
+│    │    │   └─ ....wav
+│    │    └─ ...
+```
+                            """
+        self.preprocess='''
+您可以在预处理之前修改配置文件 `config/<model_name>.yaml`，默认配置适用于GTX-1660 显卡训练 44.1khz 高采样率合成器。
+### 备注：
+1. 请保持所有音频切片的采样率与 yaml 配置文件中的采样率一致！如果不一致，程序可以跑，但训练过程中的重新采样将非常缓慢。（可选：使用Adobe Audition™的响度匹配功能可以一次性完成重采样修改声道和响度匹配。）
+
+2. 训练数据集的音频切片���数建议为约 1000 个，另外长音频切成小段可以加快训练速度，但所有音频切片的时长不应少于 2 秒。如果音频切片太多，则需要较大的内存，配置文件中将 `cache_all_data` 选项设置为 false 可以解决此问题。
+
+3. 验证集的音频切片总数建议为 10 个左右，不要放太多，不然验证过程会很慢。
+
+4. 如果您的数据集质量不是很高，请在配置文件中将 'f0_extractor' 设为 'crepe'。crepe 算法的抗噪性最好，但代价是会极大增加数据预处理所需的时间。
+
+5. 配置文件中的 ‘n_spk’ 参数将控制是否训练多说话人模型。如果您要训练**多说话人**模型，为了对说话人进行编号，所有音频文件夹的名称必须是**不大于 ‘n_spk’ 的正整数**。
+        '''
+        self.train='''
+## 训练
+
+### 1. 不使用预训练数据进行训练：
+```bash
+# 以训练 combsub 模型为例 
+python train.py -c configs/combsub.yaml
+```
+1. 训练其他模型方法类似。
+
+2. 可以随时中止训练，然后运行相同的命令来继续训练。
+
+3. 微调 (finetune)：在中止训练后，重新预处理新数据集或更改训练参数（batchsize、lr等），然后运行相同的命令。
+### 2. 使用预训练数据（底模）进行训练：
+1. **使用预训练模型请修改配置文件中的 'n_spk' 参数为 '2' ,同时配置`train`目录结构为多人物目录，不论你是否训练多说话人模型。**
+2. **如果你要训练一个更多说话人的模型，就不要下载预训练模型了。**
+3. 欢迎PR训练的多人底模 (请使用授权同意开源的数据集进行训练)。
+4. 从[**这里**](https://github.com/yxlllc/DDSP-SVC/releases/download/2.0/opencpop+kiritan.zip)下载预训练模型，并将`model_300000.pt`解压到`.\exp\combsub-test\`中
+5. 同不使用预训练数据进行训练一样，启动训练。
+        '''
+        self.visualize='''
+## 可视化
+```bash
+# 使用tensorboard检查训练状态
+tensorboard --logdir=exp
+```
+第一次验证 (validation) 后，在 TensorBoard 中可以看到合成后的测试音频。
+
+注：TensorBoard 中的测试音频是 DDSP-SVC 模型的原始输出，并未通过增强器增强。
+        '''
+        self.infer='''
+## 非实时变声
+1. （**推荐**）使用预训练声码器增强 DDSP 的输出结果：
+```bash
+# 默认 enhancer_adaptive_key = 0 正常音域范围内将有更高的音质
+# 设置 enhancer_adaptive_key > 0 可将增强器适配于更高的音域
+python main.py -i <input.wav> -m <model_file.pt> -o <output.wav> -k <keychange (semitones)> -id <speaker_id> -e true -eak <enhancer_adaptive_key (semitones)>
+```
+2. DDSP 的原始输出结果：
+```bash
+# 速度快，但音质相对较低（像您在tensorboard里听到的那样）
+python main.py -i <input.wav> -m <model_file.pt> -o <output.wav> -k <keychange (semitones)> -e false -id <speaker_id>
+```
+3. 关于 f0 提取器、响应阈值及其他参数，参见:
+
+```bash
+python main.py -h
+```
+4. 如果要使用混合说话人（捏音色）功能，增添 “-mix” 选项来设计音色，下面是个例子：
+```bash
+# 将1号说话人和2号说话人的音色按照0.5:0.5的比例混合
+python main.py -i <input.wav> -m <model_file.pt> -o <output.wav> -k <keychange (semitones)> -mix "{1:0.5, 2:0.5}" -e true -eak 0
+```
+        '''
+
+
+
+
+webui=WebUI()