Merge pull request #1618 from CNChTu/main

add fcpe for realtime

gui_v1.py

@@ -122,6 +122,7 @@ if __name__ == "__main__":
                     data["harvest"] = data["f0method"] == "harvest"
                     data["crepe"] = data["f0method"] == "crepe"
                     data["rmvpe"] = data["f0method"] == "rmvpe"
+                    data["fcpe"] = data["f0method"] == "fcpe"
                     if data["sg_input_device"] not in input_devices:
                         data["sg_input_device"] = input_devices[sd.default.device[0]]
                     if data["sg_output_device"] not in output_devices:
@@ -147,6 +148,7 @@ if __name__ == "__main__":
                     data["harvest"] = data["f0method"] == "harvest"
                     data["crepe"] = data["f0method"] == "crepe"
                     data["rmvpe"] = data["f0method"] == "rmvpe"
+                    data["fcpe"] = data["f0method"] == "fcpe"
             return data
 
         def launcher(self):
@@ -287,6 +289,13 @@ if __name__ == "__main__":
                                     default=data.get("rmvpe", "") == True,
                                     enable_events=True,
                                 ),
+                                sg.Radio(
+                                    "fcpe",
+                                    "f0method",
+                                    key="fcpe",
+                                    default=data.get("fcpe", "") == True,
+                                    enable_events=True,
+                                ),
                             ],
                         ],
                         title=i18n("常规设置"),
@@ -445,12 +454,13 @@ if __name__ == "__main__":
                             "n_cpu": values["n_cpu"],
                             # "use_jit": values["use_jit"],
                             "use_jit": False,
-                            "f0method": ["pm", "harvest", "crepe", "rmvpe"][
+                            "f0method": ["pm", "harvest", "crepe", "rmvpe", "fcpe"][
                                 [
                                     values["pm"],
                                     values["harvest"],
                                     values["crepe"],
                                     values["rmvpe"],
+                                    values["fcpe"],
                                 ].index(True)
                             ],
                         }
@@ -484,7 +494,7 @@ if __name__ == "__main__":
                         self.rvc.change_index_rate(values["index_rate"])
                 elif event == "rms_mix_rate":
                     self.gui_config.rms_mix_rate = values["rms_mix_rate"]
-                elif event in ["pm", "harvest", "crepe", "rmvpe"]:
+                elif event in ["pm", "harvest", "crepe", "rmvpe", "fcpe"]:
                     self.gui_config.f0method = event
                 elif event == "I_noise_reduce":
                     self.gui_config.I_noise_reduce = values["I_noise_reduce"]
@@ -531,12 +541,13 @@ if __name__ == "__main__":
             self.gui_config.rms_mix_rate = values["rms_mix_rate"]
             self.gui_config.index_rate = values["index_rate"]
             self.gui_config.n_cpu = values["n_cpu"]
-            self.gui_config.f0method = ["pm", "harvest", "crepe", "rmvpe"][
+            self.gui_config.f0method = ["pm", "harvest", "crepe", "rmvpe", "fcpe"][
                 [
                     values["pm"],
                     values["harvest"],
                     values["crepe"],
                     values["rmvpe"],
+                    values["fcpe"],
                 ].index(True)
             ]
             return True

requirements.txt

@@ -42,6 +42,7 @@ onnxruntime; sys_platform == 'darwin'
 onnxruntime-gpu; sys_platform != 'darwin'
 torchcrepe==0.0.20
 fastapi==0.88
+torchfcpe
 ffmpy==0.3.1
 python-dotenv>=1.0.0
 av

tools/rvc_for_realtime.py

@@ -62,7 +62,6 @@ class RVC:
         """
         try:
             if config.dml == True:
-
                 def forward_dml(ctx, x, scale):
                     ctx.scale = scale
                     res = x.clone().detach()
@@ -183,6 +182,8 @@ class RVC:
 
             if last_rvc is not None and hasattr(last_rvc, "model_rmvpe"):
                 self.model_rmvpe = last_rvc.model_rmvpe
+            if last_rvc is not None and hasattr(last_rvc, "model_fcpe"):
+                self.model_fcpe = last_rvc.model_fcpe
         except:
             printt(traceback.format_exc())
 
@@ -217,6 +218,8 @@ class RVC:
             return self.get_f0_crepe(x, f0_up_key)
         if method == "rmvpe":
             return self.get_f0_rmvpe(x, f0_up_key)
+        if method == "fcpe":
+            return self.get_f0_fcpe(x, f0_up_key)
         if method == "pm":
             p_len = x.shape[0] // 160 + 1
             f0_min = 65
@@ -322,6 +325,20 @@ class RVC:
         f0 *= pow(2, f0_up_key / 12)
         return self.get_f0_post(f0)
 
+    def get_f0_fcpe(self, x, f0_up_key):
+        if hasattr(self, "model_fcpe") == False:
+            from torchfcpe import spawn_bundled_infer_model
+            printt("Loading fcpe model")
+            self.model_fcpe = spawn_bundled_infer_model(self.device)
+        f0 = self.model_fcpe.infer(
+            torch.from_numpy(x).to(self.device).unsqueeze(0).float(),
+            sr=16000,
+            decoder_mode='local_argmax',
+            threshold=0.006,
+        ).squeeze().cpu().numpy()
+        f0 *= pow(2, f0_up_key / 12)
+        return self.get_f0_post(f0)
+
     def infer(
             self,
             feats: torch.Tensor,