Automatic Speech Recognition
Transformers
NeMo
Safetensors
parakeet_tdt
feature-extraction
speech-recognition
parakeet
tdt
fastconformer
Eval Results
Instructions to use FluidVoice-Labs/fluid-speech-pro with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use FluidVoice-Labs/fluid-speech-pro with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("automatic-speech-recognition", model="FluidVoice-Labs/fluid-speech-pro")# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("FluidVoice-Labs/fluid-speech-pro", device_map="auto") - NeMo
How to use FluidVoice-Labs/fluid-speech-pro with NeMo:
import nemo.collections.asr as nemo_asr asr_model = nemo_asr.models.ASRModel.from_pretrained("FluidVoice-Labs/fluid-speech-pro") transcriptions = asr_model.transcribe(["file.wav"]) - Notebooks
- Google Colab
- Kaggle
Download fluid_speech.py from FluidVoice-Labs/fluid-speech-pro: direct link, hf CLI and curl.
- Browser
- Download file 4.68 kB
-
https://huggingface.co/FluidVoice-Labs/fluid-speech-pro/resolve/main/fluid_speech.py
- Command line
-
hf download hf://FluidVoice-Labs/fluid-speech-pro/fluid_speech.py
-
curl -L -o fluid_speech.py https://huggingface.co/FluidVoice-Labs/fluid-speech-pro/resolve/main/fluid_speech.py
4.68 kB
| """Load Fluid Speech: NVIDIA Parakeet TDT 0.6B v3 with one small extra block between subsampling and the first | |
| Conformer layer. The .nemo file holds that block's weights next to the usual ones, so a plain | |
| `ASRModel.from_pretrained` / `restore_from` stops with "Unexpected key(s) ... encoder.fv_insert" instead of quietly | |
| running without it; `load()` below adds the block and then loads every tensor strictly. | |
| Clean mode is an optional transcription style for edited instead of verbatim text (the style of SPGISpeech's | |
| references): it removes repeated words and filler words. Its checkpoint is merged into the model checkpoint, so | |
| every dataset loads the same file; `load(..., clean_mode=True)` switches it on by adding its weights into the model's | |
| before transcribing. | |
| import fluid_speech | |
| model = fluid_speech.load("<org>/<model>") # Hub repo id, or a local .nemo path | |
| model = fluid_speech.load("<org>/<model>", clean_mode=True) # edited, non-verbatim text | |
| """ | |
| import os | |
| import tarfile | |
| import tempfile | |
| import torch | |
| import torch.nn as nn | |
| from nemo.collections.asr.models import ASRModel | |
| CLEAN_MODE_PREFIX = "clean_mode." | |
| class FrameCleaner(nn.Module): | |
| """Residual block on the subsampled frames: LayerNorm, projection, depthwise time convolution, GELU, projection.""" | |
| def __init__(self, width=1024, dim=1024, kernel=15): | |
| super().__init__() | |
| self.ln = nn.LayerNorm(width) | |
| self.down = nn.Linear(width, dim) | |
| self.mix = nn.Conv1d(dim, dim, kernel, padding=kernel // 2, groups=dim) | |
| self.up = nn.Linear(dim, width) | |
| def forward(self, x): # x: [batch, time, width] | |
| h = self.down(self.ln(x)).transpose(1, 2) | |
| return x + self.up(nn.functional.gelu(self.mix(h)).transpose(1, 2)) | |
| def attach(model, dim, kernel): | |
| """Add the block to `model.encoder` and run it on the output of the subsampling module.""" | |
| encoder = model.encoder | |
| encoder.fv_insert = FrameCleaner(encoder.d_model, dim, kernel).to(next(encoder.parameters()).device) | |
| def clean_frames(module, args, out): | |
| if isinstance(out, tuple): | |
| return (encoder.fv_insert(out[0]),) + tuple(out[1:]) | |
| return encoder.fv_insert(out) | |
| encoder.pre_encode.register_forward_hook(clean_frames) | |
| return encoder.fv_insert | |
| def state_dict_of(nemo_path): | |
| with tarfile.open(nemo_path) as tar, tempfile.TemporaryDirectory() as tmp: | |
| member = next(m for m in tar.getmembers() if os.path.basename(m.name) == "model_weights.ckpt") | |
| tar.extract(member, tmp, filter="data") | |
| return torch.load(os.path.join(tmp, member.name), map_location="cpu", weights_only=True) | |
| def resolve(model_id): | |
| if os.path.exists(model_id): | |
| return model_id | |
| from huggingface_hub import hf_hub_download, list_repo_files | |
| nemo_files = [f for f in list_repo_files(model_id) if f.endswith(".nemo")] | |
| assert len(nemo_files) == 1, f"expected one .nemo file in {model_id}, found {nemo_files}" | |
| return hf_hub_download(model_id, nemo_files[0]) | |
| def split_clean_mode(state): | |
| """Take the adapter's tensors (`clean_mode.<module>.A` / `.B`) out of the checkpoint's state dict.""" | |
| adapter = {k[len(CLEAN_MODE_PREFIX):]: state.pop(k) for k in [k for k in state if k.startswith(CLEAN_MODE_PREFIX)]} | |
| return state, adapter | |
| def fold_clean_mode(state, adapter, scale): | |
| """Add scale * B @ A to every adapted encoder weight, and clean mode's two input vectors to the biases they meet.""" | |
| decoder_input, joint_hidden = adapter.pop("decoder_input"), adapter.pop("joint_hidden") | |
| lstm = "decoder.prediction.dec_rnn.lstm" | |
| state[f"{lstm}.bias_ih_l0"] = state[f"{lstm}.bias_ih_l0"] + state[f"{lstm}.weight_ih_l0"] @ decoder_input | |
| state["joint.pred.bias"] = state["joint.pred.bias"] + joint_hidden | |
| for name in sorted({k.rsplit(".", 1)[0] for k in adapter}): | |
| key = f"encoder.{name}.weight" | |
| assert key in state, f"clean mode adapts {key}, which the model does not have" | |
| delta = scale * (adapter[f"{name}.B"] @ adapter[f"{name}.A"]) | |
| state[key] = state[key] + delta.view_as(state[key]).to(state[key].dtype) | |
| return state | |
| def load(model_id, map_location=None, clean_mode=False): | |
| path = resolve(model_id) | |
| model = ASRModel.restore_from(path, map_location=map_location, strict=False) | |
| spec = model.cfg.fluid_speech | |
| attach(model, spec.insert_dim, spec.insert_kernel) | |
| state, adapter = split_clean_mode(state_dict_of(path)) | |
| if clean_mode: | |
| state = fold_clean_mode(state, adapter, float(spec.clean_mode.alpha) / int(spec.clean_mode.rank)) | |
| model.load_state_dict(state, strict=True) | |
| return model.eval() | |