Automatic Speech Recognition
Transformers
NeMo
Safetensors
parakeet_tdt
feature-extraction
speech-recognition
parakeet
tdt
fastconformer
Eval Results
Instructions to use FluidVoice-Labs/fluid-speech-pro with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use FluidVoice-Labs/fluid-speech-pro with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("automatic-speech-recognition", model="FluidVoice-Labs/fluid-speech-pro")# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("FluidVoice-Labs/fluid-speech-pro", device_map="auto") - NeMo
How to use FluidVoice-Labs/fluid-speech-pro with NeMo:
import nemo.collections.asr as nemo_asr asr_model = nemo_asr.models.ASRModel.from_pretrained("FluidVoice-Labs/fluid-speech-pro") transcriptions = asr_model.transcribe(["file.wav"]) - Notebooks
- Google Colab
- Kaggle
File size: 4,675 Bytes
8104e7e 53f490b 412ef0d 36481f6 8104e7e 412ef0d 8104e7e 412ef0d 8104e7e 412ef0d 8104e7e 412ef0d 36481f6 412ef0d 8104e7e 412ef0d 8104e7e 412ef0d 8104e7e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 | """Load Fluid Speech: NVIDIA Parakeet TDT 0.6B v3 with one small extra block between subsampling and the first
Conformer layer. The .nemo file holds that block's weights next to the usual ones, so a plain
`ASRModel.from_pretrained` / `restore_from` stops with "Unexpected key(s) ... encoder.fv_insert" instead of quietly
running without it; `load()` below adds the block and then loads every tensor strictly.
Clean mode is an optional transcription style for edited instead of verbatim text (the style of SPGISpeech's
references): it removes repeated words and filler words. Its checkpoint is merged into the model checkpoint, so
every dataset loads the same file; `load(..., clean_mode=True)` switches it on by adding its weights into the model's
before transcribing.
import fluid_speech
model = fluid_speech.load("<org>/<model>") # Hub repo id, or a local .nemo path
model = fluid_speech.load("<org>/<model>", clean_mode=True) # edited, non-verbatim text
"""
import os
import tarfile
import tempfile
import torch
import torch.nn as nn
from nemo.collections.asr.models import ASRModel
CLEAN_MODE_PREFIX = "clean_mode."
class FrameCleaner(nn.Module):
"""Residual block on the subsampled frames: LayerNorm, projection, depthwise time convolution, GELU, projection."""
def __init__(self, width=1024, dim=1024, kernel=15):
super().__init__()
self.ln = nn.LayerNorm(width)
self.down = nn.Linear(width, dim)
self.mix = nn.Conv1d(dim, dim, kernel, padding=kernel // 2, groups=dim)
self.up = nn.Linear(dim, width)
def forward(self, x): # x: [batch, time, width]
h = self.down(self.ln(x)).transpose(1, 2)
return x + self.up(nn.functional.gelu(self.mix(h)).transpose(1, 2))
def attach(model, dim, kernel):
"""Add the block to `model.encoder` and run it on the output of the subsampling module."""
encoder = model.encoder
encoder.fv_insert = FrameCleaner(encoder.d_model, dim, kernel).to(next(encoder.parameters()).device)
def clean_frames(module, args, out):
if isinstance(out, tuple):
return (encoder.fv_insert(out[0]),) + tuple(out[1:])
return encoder.fv_insert(out)
encoder.pre_encode.register_forward_hook(clean_frames)
return encoder.fv_insert
def state_dict_of(nemo_path):
with tarfile.open(nemo_path) as tar, tempfile.TemporaryDirectory() as tmp:
member = next(m for m in tar.getmembers() if os.path.basename(m.name) == "model_weights.ckpt")
tar.extract(member, tmp, filter="data")
return torch.load(os.path.join(tmp, member.name), map_location="cpu", weights_only=True)
def resolve(model_id):
if os.path.exists(model_id):
return model_id
from huggingface_hub import hf_hub_download, list_repo_files
nemo_files = [f for f in list_repo_files(model_id) if f.endswith(".nemo")]
assert len(nemo_files) == 1, f"expected one .nemo file in {model_id}, found {nemo_files}"
return hf_hub_download(model_id, nemo_files[0])
def split_clean_mode(state):
"""Take the adapter's tensors (`clean_mode.<module>.A` / `.B`) out of the checkpoint's state dict."""
adapter = {k[len(CLEAN_MODE_PREFIX):]: state.pop(k) for k in [k for k in state if k.startswith(CLEAN_MODE_PREFIX)]}
return state, adapter
def fold_clean_mode(state, adapter, scale):
"""Add scale * B @ A to every adapted encoder weight, and clean mode's two input vectors to the biases they meet."""
decoder_input, joint_hidden = adapter.pop("decoder_input"), adapter.pop("joint_hidden")
lstm = "decoder.prediction.dec_rnn.lstm"
state[f"{lstm}.bias_ih_l0"] = state[f"{lstm}.bias_ih_l0"] + state[f"{lstm}.weight_ih_l0"] @ decoder_input
state["joint.pred.bias"] = state["joint.pred.bias"] + joint_hidden
for name in sorted({k.rsplit(".", 1)[0] for k in adapter}):
key = f"encoder.{name}.weight"
assert key in state, f"clean mode adapts {key}, which the model does not have"
delta = scale * (adapter[f"{name}.B"] @ adapter[f"{name}.A"])
state[key] = state[key] + delta.view_as(state[key]).to(state[key].dtype)
return state
def load(model_id, map_location=None, clean_mode=False):
path = resolve(model_id)
model = ASRModel.restore_from(path, map_location=map_location, strict=False)
spec = model.cfg.fluid_speech
attach(model, spec.insert_dim, spec.insert_kernel)
state, adapter = split_clean_mode(state_dict_of(path))
if clean_mode:
state = fold_clean_mode(state, adapter, float(spec.clean_mode.alpha) / int(spec.clean_mode.rank))
model.load_state_dict(state, strict=True)
return model.eval()
|