Merge pull request #3494 from wuhongsheng/silero_vad

feat(vad): add Silero VAD adapter

Author: LauraGPT

README.md

@@ -189,6 +189,12 @@ from funasr import AutoModel
 model = AutoModel(model="paraformer-zh", vad_model="fsmn-vad", punc_model="ct-punc", spk_model="cam++", device="cuda")
 result = model.generate(input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav", hotword="关键词 20")
 
+# Optional Silero VAD (install first: python -m pip install "funasr[silero]")
+model = AutoModel(
+    model="paraformer-zh", vad_model="silero-vad", device="cuda",
+    vad_kwargs={"silero_threshold": 0.5, "silero_min_silence_duration_ms": 100},
+)
+result = model.generate(input="audio.wav")
 
 # Streaming real-time (feed audio chunk by chunk)
 import soundfile as sf

README_zh.md

@@ -239,6 +239,13 @@ from funasr import AutoModel
 model = AutoModel(model="paraformer-zh", vad_model="fsmn-vad", punc_model="ct-punc", spk_model="cam++", device="cuda")
 result = model.generate(input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav", hotword="关键词 20")
 
+# 使用 Silero VAD（先安装：python -m pip install "funasr[silero]"）
+model = AutoModel(
+    model="paraformer-zh", vad_model="silero-vad", device="cpu",
+    vad_kwargs={"silero_threshold": 0.5, "silero_min_silence_duration_ms": 100},
+)
+result = model.generate(input="audio.wav")
+
 # 中/英/日 + 中文方言
 model = AutoModel(model="FunAudioLLM/Fun-ASR-Nano-2512", hub="hf", trust_remote_code=True,
                   vad_model="fsmn-vad", vad_kwargs={"max_single_segment_time": 30000}, device="cuda")

funasr/auto/auto_model.py

@@ -536,6 +536,12 @@ def build_model(**kwargs):
                 kwargs contains the resolved configuration.
         """
         assert "model" in kwargs
+        # Silero VAD is loaded by its optional Python package rather than a
+        # FunASR model repository. Supplying model_conf keeps it on the normal
+        # AutoModel construction path while bypassing hub config resolution.
+        if kwargs["model"] in {"silero-vad", "silero_vad"}:
+            kwargs.setdefault("model_conf", {})
+            kwargs["model"] = "SileroVad"
         if "model_conf" not in kwargs:
             logging.info("download models from model hub: {}".format(kwargs.get("hub", "ms")))
             kwargs = download_model(**kwargs)

funasr/models/silero_vad/__init__.py

@@ -0,0 +1 @@
+"""Silero VAD adapter for the FunASR AutoModel pipeline."""

funasr/models/silero_vad/model.py

@@ -0,0 +1,97 @@
+"""Adapter that makes Silero VAD return FunASR-compatible millisecond segments."""
+
+import time
+
+import torch
+
+from funasr.register import tables
+from funasr.utils.load_utils import load_audio_text_image_video
+
+
+@tables.register("model_classes", "SileroVad")
+class SileroVad(torch.nn.Module):
+    """Offline Silero VAD adapter used by ``AutoModel(vad_model='silero-vad')``.
+
+    Requires the official ``silero-vad`` Python package.
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__()
+        self.anchor = torch.nn.Parameter(torch.empty(0), requires_grad=False)
+        try:
+            from silero_vad import get_speech_timestamps, load_silero_vad
+        except ImportError as error:
+            raise ImportError(
+                "Silero VAD requires the optional dependency. Install it with "
+                '`python -m pip install "funasr[silero]"` or '
+                "`python -m pip install silero-vad`."
+            ) from error
+        self.onnx = bool(kwargs.get("silero_onnx", False))
+        self.model = load_silero_vad(onnx=self.onnx)
+        self.get_speech_timestamps = get_speech_timestamps
+
+    @staticmethod
+    def _split_long_segments(segments, max_single_segment_time):
+        if not max_single_segment_time:
+            return segments
+        limit_ms = int(max_single_segment_time)
+        if limit_ms <= 0:
+            raise ValueError(
+                "max_single_segment_time must resolve to a positive millisecond value"
+            )
+        split = []
+        for start, end in segments:
+            while end - start > limit_ms:
+                split.append([start, start + limit_ms])
+                start += limit_ms
+            split.append([start, end])
+        return split
+
+    def inference(self, data_in, key=None, **kwargs):
+        sample_rate = int(kwargs.get("silero_sampling_rate", 16000))
+        if sample_rate not in (8000, 16000):
+            raise ValueError("Silero VAD supports silero_sampling_rate=8000 or 16000")
+        audio_list = load_audio_text_image_video(
+            data_in,
+            fs=sample_rate,
+            audio_fs=kwargs.get("fs", sample_rate),
+            data_type=kwargs.get("data_type", "sound"),
+        )
+        if not isinstance(audio_list, list):
+            audio_list = [audio_list]
+
+        started = time.perf_counter()
+        results = []
+        for index, audio in enumerate(audio_list):
+            device = torch.device("cpu") if self.onnx else self.anchor.device
+            waveform = torch.as_tensor(audio, dtype=torch.float32).flatten().to(device)
+            timestamps = self.get_speech_timestamps(
+                waveform,
+                self.model,
+                sampling_rate=sample_rate,
+                threshold=kwargs.get("silero_threshold", 0.5),
+                min_speech_duration_ms=kwargs.get("silero_min_speech_duration_ms", 250),
+                min_silence_duration_ms=kwargs.get(
+                    "silero_min_silence_duration_ms", 100
+                ),
+                speech_pad_ms=kwargs.get("silero_speech_pad_ms", 30),
+            )
+            segments = [
+                [
+                    int(item["start"] * 1000 / sample_rate),
+                    int(item["end"] * 1000 / sample_rate),
+                ]
+                for item in timestamps
+            ]
+            segments = self._split_long_segments(
+                segments, kwargs.get("max_single_segment_time")
+            )
+            results.append(
+                {"key": key[index] if key else str(index), "value": segments}
+            )
+        elapsed = time.perf_counter() - started
+        total_samples = sum(len(torch.as_tensor(audio)) for audio in audio_list)
+        return results, {
+            "batch_data_time": total_samples / sample_rate,
+            "forward": elapsed,
+        }

setup.py

@@ -48,6 +48,9 @@
     "train": [
         "rapidfuzz>=3.0.0",
     ],
+    "silero": [
+        "silero-vad>=6.0.0",
+    ],
     # all: The modules should be optionally installled due to some reason.
     #      Please consider moving them to "install" occasionally
     "all": [

tests/test_silero_vad_adapter.py

@@ -0,0 +1,100 @@
+import unittest
+from importlib.util import find_spec
+from unittest.mock import patch
+
+import torch
+
+from funasr.auto.auto_model import AutoModel
+from funasr.models.silero_vad.model import SileroVad
+
+
+@unittest.skipUnless(find_spec("silero_vad"), "silero-vad is not installed")
+class TestSileroVadAdapter(unittest.TestCase):
+    def _timestamps_stub(self, waveform, model, sampling_rate, **options):
+        self.assertEqual(sampling_rate, 16000)
+        self.assertEqual(options["threshold"], 0.6)
+        return [{"start": 1600, "end": 17600}]
+
+    def _load_stub(self, *args, **kwargs):
+        self.assertEqual(kwargs, {"onnx": False})
+        return torch.nn.Identity()
+
+    @patch("silero_vad.get_speech_timestamps")
+    @patch("silero_vad.load_silero_vad")
+    def test_returns_funasr_millisecond_segments_and_honors_max_length(
+        self, load_model, timestamps
+    ):
+        load_model.side_effect = self._load_stub
+        timestamps.side_effect = self._timestamps_stub
+        model = SileroVad()
+        results, metadata = model.inference(
+            data_in=[torch.zeros(32000)],
+            key=["sample"],
+            silero_threshold=0.6,
+            max_single_segment_time=500,
+        )
+
+        self.assertEqual(
+            results, [{"key": "sample", "value": [[100, 600], [600, 1100]]}]
+        )
+        self.assertEqual(metadata["batch_data_time"], 2.0)
+        load_model.assert_called_once_with(onnx=False)
+
+    @patch("silero_vad.get_speech_timestamps")
+    @patch("silero_vad.load_silero_vad")
+    def test_rejects_unsupported_sampling_rate(self, load_model, timestamps):
+        load_model.side_effect = self._load_stub
+        model = SileroVad()
+        with self.assertRaisesRegex(ValueError, "8000 or 16000"):
+            model.inference(data_in=[torch.zeros(16000)], silero_sampling_rate=44100)
+
+    @patch("silero_vad.get_speech_timestamps")
+    @patch("silero_vad.load_silero_vad")
+    def test_auto_model_alias_uses_the_existing_vad_build_path(
+        self, load_model, timestamps
+    ):
+        load_model.side_effect = self._load_stub
+        model, resolved = AutoModel.build_model(model="silero-vad", device="cpu")
+        self.assertIsInstance(model, SileroVad)
+        self.assertEqual(resolved["model"], "SileroVad")
+
+    @patch("silero_vad.get_speech_timestamps")
+    @patch("silero_vad.load_silero_vad")
+    def test_waveform_follows_the_adapter_device(self, load_model, timestamps):
+        load_model.side_effect = self._load_stub
+
+        def timestamps_stub(waveform, model, sampling_rate, **options):
+            self.assertEqual(waveform.device.type, "meta")
+            return []
+
+        timestamps.side_effect = timestamps_stub
+        model = SileroVad().to("meta")
+        results, _ = model.inference(data_in=[torch.zeros(16000)], key=["sample"])
+        self.assertEqual(results, [{"key": "sample", "value": []}])
+
+    @patch("silero_vad.get_speech_timestamps")
+    @patch("silero_vad.load_silero_vad")
+    def test_onnx_waveform_stays_on_cpu(self, load_model, timestamps):
+        load_model.return_value = object()
+
+        def timestamps_stub(waveform, model, sampling_rate, **options):
+            self.assertEqual(waveform.device.type, "cpu")
+            return []
+
+        timestamps.side_effect = timestamps_stub
+        model = SileroVad(silero_onnx=True).to("meta")
+        results, _ = model.inference(data_in=[torch.zeros(16000)], key=["sample"])
+        self.assertEqual(results, [{"key": "sample", "value": []}])
+        load_model.assert_called_once_with(onnx=True)
+
+    def test_rejects_negative_max_segment_length(self):
+        with self.assertRaisesRegex(ValueError, "positive"):
+            SileroVad._split_long_segments([[100, 1100]], -500)
+
+    def test_rejects_sub_millisecond_max_segment_length(self):
+        with self.assertRaisesRegex(ValueError, "positive"):
+            SileroVad._split_long_segments([[100, 1100]], 0.5)
+
+
+if __name__ == "__main__":
+    unittest.main()