157 lines
5.4 KiB
Python
157 lines
5.4 KiB
Python
import io
|
|
import logging
|
|
import os
|
|
|
|
import librosa
|
|
import numpy as np
|
|
import parselmouth
|
|
import soundfile
|
|
import torch
|
|
import torchaudio
|
|
|
|
import utils
|
|
from inference import slicer
|
|
from models import SynthesizerTrn
|
|
|
|
logging.getLogger('numba').setLevel(logging.WARNING)
|
|
logging.getLogger('matplotlib').setLevel(logging.WARNING)
|
|
|
|
def resize2d_f0(x, target_len):
|
|
source = np.array(x)
|
|
source[source < 0.001] = np.nan
|
|
target = np.interp(np.arange(0, len(source) * target_len, len(source)) / target_len, np.arange(0, len(source)),
|
|
source)
|
|
res = np.nan_to_num(target)
|
|
return res
|
|
|
|
def get_f0(x, p_len,f0_up_key=0):
|
|
|
|
time_step = 160 / 16000 * 1000
|
|
f0_min = 50
|
|
f0_max = 1100
|
|
f0_mel_min = 1127 * np.log(1 + f0_min / 700)
|
|
f0_mel_max = 1127 * np.log(1 + f0_max / 700)
|
|
|
|
f0 = parselmouth.Sound(x, 16000).to_pitch_ac(
|
|
time_step=time_step / 1000, voicing_threshold=0.6,
|
|
pitch_floor=f0_min, pitch_ceiling=f0_max).selected_array['frequency']
|
|
|
|
pad_size=(p_len - len(f0) + 1) // 2
|
|
if(pad_size>0 or p_len - len(f0) - pad_size>0):
|
|
f0 = np.pad(f0,[[pad_size,p_len - len(f0) - pad_size]], mode='constant')
|
|
|
|
f0 *= pow(2, f0_up_key / 12)
|
|
f0_mel = 1127 * np.log(1 + f0 / 700)
|
|
f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / (f0_mel_max - f0_mel_min) + 1
|
|
f0_mel[f0_mel <= 1] = 1
|
|
f0_mel[f0_mel > 255] = 255
|
|
f0_coarse = np.rint(f0_mel).astype(np.int)
|
|
return f0_coarse, f0
|
|
|
|
def clean_pitch(input_pitch):
|
|
num_nan = np.sum(input_pitch == 1)
|
|
if num_nan / len(input_pitch) > 0.9:
|
|
input_pitch[input_pitch != 1] = 1
|
|
return input_pitch
|
|
|
|
|
|
def plt_pitch(input_pitch):
|
|
input_pitch = input_pitch.astype(float)
|
|
input_pitch[input_pitch == 1] = np.nan
|
|
return input_pitch
|
|
|
|
|
|
def f0_to_pitch(ff):
|
|
f0_pitch = 69 + 12 * np.log2(ff / 440)
|
|
return f0_pitch
|
|
|
|
|
|
def fill_a_to_b(a, b):
|
|
if len(a) < len(b):
|
|
for _ in range(0, len(b) - len(a)):
|
|
a.append(a[0])
|
|
|
|
|
|
def mkdir(paths: list):
|
|
for path in paths:
|
|
if not os.path.exists(path):
|
|
os.mkdir(path)
|
|
|
|
|
|
class VitsSvc(object):
|
|
def __init__(self):
|
|
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
|
self.SVCVITS = None
|
|
self.hps = None
|
|
self.speakers = None
|
|
self.hubert_soft = utils.get_hubert_model()
|
|
|
|
def set_device(self, device):
|
|
self.device = torch.device(device)
|
|
self.hubert_soft.to(self.device)
|
|
if self.SVCVITS is not None:
|
|
self.SVCVITS.to(self.device)
|
|
|
|
def loadCheckpoint(self, path):
|
|
self.hps = utils.get_hparams_from_file(f"checkpoints/{path}/config.json")
|
|
self.SVCVITS = SynthesizerTrn(
|
|
self.hps.data.filter_length // 2 + 1,
|
|
self.hps.train.segment_size // self.hps.data.hop_length,
|
|
**self.hps.model)
|
|
_ = utils.load_checkpoint(f"checkpoints/{path}/model.pth", self.SVCVITS, None)
|
|
_ = self.SVCVITS.eval().to(self.device)
|
|
self.speakers = self.hps.spk
|
|
|
|
def get_units(self, source, sr):
|
|
source = source.unsqueeze(0).to(self.device)
|
|
with torch.inference_mode():
|
|
units = self.hubert_soft.units(source)
|
|
return units
|
|
|
|
|
|
def get_unit_pitch(self, in_path, tran):
|
|
source, sr = torchaudio.load(in_path)
|
|
source = torchaudio.functional.resample(source, sr, 16000)
|
|
if len(source.shape) == 2 and source.shape[1] >= 2:
|
|
source = torch.mean(source, dim=0).unsqueeze(0)
|
|
soft = self.get_units(source, sr).squeeze(0).cpu().numpy()
|
|
f0_coarse, f0 = get_f0(source.cpu().numpy()[0], soft.shape[0]*2, tran)
|
|
return soft, f0
|
|
|
|
def infer(self, speaker_id, tran, raw_path):
|
|
speaker_id = self.speakers[speaker_id]
|
|
sid = torch.LongTensor([int(speaker_id)]).to(self.device).unsqueeze(0)
|
|
soft, pitch = self.get_unit_pitch(raw_path, tran)
|
|
f0 = torch.FloatTensor(clean_pitch(pitch)).unsqueeze(0).to(self.device)
|
|
stn_tst = torch.FloatTensor(soft)
|
|
with torch.no_grad():
|
|
x_tst = stn_tst.unsqueeze(0).to(self.device)
|
|
x_tst = torch.repeat_interleave(x_tst, repeats=2, dim=1).transpose(1, 2)
|
|
audio,_ = self.SVCVITS.infer(x_tst, f0=f0, g=sid)[0,0].data.float()
|
|
return audio, audio.shape[-1]
|
|
|
|
def inference(self,srcaudio,chara,tran,slice_db):
|
|
sampling_rate, audio = srcaudio
|
|
audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32)
|
|
if len(audio.shape) > 1:
|
|
audio = librosa.to_mono(audio.transpose(1, 0))
|
|
if sampling_rate != 16000:
|
|
audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=16000)
|
|
soundfile.write("tmpwav.wav", audio, 16000, format="wav")
|
|
chunks = slicer.cut("tmpwav.wav", db_thresh=slice_db)
|
|
audio_data, audio_sr = slicer.chunks2audio("tmpwav.wav", chunks)
|
|
audio = []
|
|
for (slice_tag, data) in audio_data:
|
|
length = int(np.ceil(len(data) / audio_sr * self.hps.data.sampling_rate))
|
|
raw_path = io.BytesIO()
|
|
soundfile.write(raw_path, data, audio_sr, format="wav")
|
|
raw_path.seek(0)
|
|
if slice_tag:
|
|
_audio = np.zeros(length)
|
|
else:
|
|
out_audio, out_sr = self.infer(chara, tran, raw_path)
|
|
_audio = out_audio.cpu().numpy()
|
|
audio.extend(list(_audio))
|
|
audio = (np.array(audio) * 32768.0).astype('int16')
|
|
return (self.hps.data.sampling_rate,audio)
|