Update training & fine-tuning instructions

Author: Plachtaa

app.py

@@ -1,3 +1,5 @@
+import os
+os.environ['HF_HUB_CACHE'] = './checkpoints/hf_cache'
 import gradio as gr
 import torch
 import torchaudio
@@ -28,6 +30,8 @@
     model[key].to(device)
 model.cfm.estimator.setup_caches(max_batch_size=1, max_seq_length=8192)
 
+print(f"cfm has {sum(p.numel() for p in model.cfm.parameters() if p.requires_grad)} trainable parameters")
+
 # Load additional modules
 from modules.campplus.DTDNN import CAMPPlus
 
@@ -75,7 +79,7 @@
     "num_mels": config['preprocess_params']['spect_params']['n_mels'],
     "sampling_rate": sr,
     "fmin": 0,
-    "fmax": None,
+    "fmax": None if config['preprocess_params']['spect_params'].get('fmax') == "None" else config['preprocess_params']['spect_params']['fmax'],
     "center": False
 }
 from modules.audio import mel_spectrogram
@@ -84,7 +88,7 @@
 
 # f0 conditioned model
 dit_checkpoint_path, dit_config_path = load_custom_model_from_hf("Plachta/Seed-VC",
-                                                "DiT_seed_v2_uvit_whisper_base_f0_44k_bigvgan_pruned_ft_ema.pth",
+                                                "DiT_seed_v2_uvit_whisper_base_f0_44k_bigvgan_pruned_ft_ema_v2.pth",
                                                 "config_dit_mel_seed_uvit_whisper_base_f0_44k.yml")
 
 config = yaml.safe_load(open(dit_config_path, 'r'))
@@ -114,7 +118,7 @@
     "num_mels": config['preprocess_params']['spect_params']['n_mels'],
     "sampling_rate": sr,
     "fmin": 0,
-    "fmax": None,
+    "fmax": None if config['preprocess_params']['spect_params'].get('fmax') == "None" else config['preprocess_params']['spect_params']['fmax'],
     "center": False
 }
 to_mel_f0 = lambda x: mel_spectrogram(x, **mel_fn_args_f0)
@@ -124,6 +128,15 @@
 bigvgan_44k_model.remove_weight_norm()
 bigvgan_44k_model = bigvgan_44k_model.eval().to(device)
 
+from modules.hifigan.generator import HiFTGenerator
+from modules.hifigan.f0_predictor import ConvRNNF0Predictor
+
+hift_config = yaml.safe_load(open('configs/hifigan.yml', 'r'))
+hift_gen = HiFTGenerator(**hift_config['hift'], f0_predictor=ConvRNNF0Predictor(**hift_config['f0_predictor']))
+hift_gen.load_state_dict(torch.load(hift_config['pretrained_model_path'], map_location='cpu'))
+hift_gen.eval()
+hift_gen.to(device)
+
 def adjust_f0_semitones(f0_sequence, n_semitones):
     factor = 2 ** (n_semitones / 12)
     return f0_sequence * factor
@@ -148,7 +161,7 @@ def crossfade(chunk1, chunk2, overlap):
 def voice_conversion(source, target, diffusion_steps, length_adjust, inference_cfg_rate, f0_condition, auto_f0_adjust, pitch_shift):
     inference_module = model if not f0_condition else model_f0
     mel_fn = to_mel if not f0_condition else to_mel_f0
-    bigvgan_fn = bigvgan_model if not f0_condition else bigvgan_44k_model
+    bigvgan_fn = hift_gen if not f0_condition else bigvgan_44k_model
     sr = 22050 if not f0_condition else 44100
     # Load audio
     source_audio = librosa.load(source, sr=sr)[0]
@@ -289,6 +302,8 @@ def voice_conversion(source, target, diffusion_steps, length_adjust, inference_c
                                                    inference_cfg_rate=inference_cfg_rate)
         vc_target = vc_target[:, :, mel2.size(-1):]
         vc_wave = bigvgan_fn(vc_target)[0]
+        if vc_wave.ndim == 1:
+            vc_wave = vc_wave.unsqueeze(0)
         if processed_frames == 0:
             if is_last_chunk:
                 output_wave = vc_wave[0].cpu().numpy()