fluid-speech-pro / fluid_speech.py
ALTICDEV's picture
Update the model and add its leaderboard results
36481f6 verified
Raw History Blame Contribute Delete
4.68 kB
"""Load Fluid Speech: NVIDIA Parakeet TDT 0.6B v3 with one small extra block between subsampling and the first
Conformer layer. The .nemo file holds that block's weights next to the usual ones, so a plain
`ASRModel.from_pretrained` / `restore_from` stops with "Unexpected key(s) ... encoder.fv_insert" instead of quietly
running without it; `load()` below adds the block and then loads every tensor strictly.
Clean mode is an optional transcription style for edited instead of verbatim text (the style of SPGISpeech's
references): it removes repeated words and filler words. Its checkpoint is merged into the model checkpoint, so
every dataset loads the same file; `load(..., clean_mode=True)` switches it on by adding its weights into the model's
before transcribing.
import fluid_speech
model = fluid_speech.load("<org>/<model>") # Hub repo id, or a local .nemo path
model = fluid_speech.load("<org>/<model>", clean_mode=True) # edited, non-verbatim text
"""
import os
import tarfile
import tempfile
import torch
import torch.nn as nn
from nemo.collections.asr.models import ASRModel
CLEAN_MODE_PREFIX = "clean_mode."
class FrameCleaner(nn.Module):
"""Residual block on the subsampled frames: LayerNorm, projection, depthwise time convolution, GELU, projection."""
def __init__(self, width=1024, dim=1024, kernel=15):
super().__init__()
self.ln = nn.LayerNorm(width)
self.down = nn.Linear(width, dim)
self.mix = nn.Conv1d(dim, dim, kernel, padding=kernel // 2, groups=dim)
self.up = nn.Linear(dim, width)
def forward(self, x): # x: [batch, time, width]
h = self.down(self.ln(x)).transpose(1, 2)
return x + self.up(nn.functional.gelu(self.mix(h)).transpose(1, 2))
def attach(model, dim, kernel):
"""Add the block to `model.encoder` and run it on the output of the subsampling module."""
encoder = model.encoder
encoder.fv_insert = FrameCleaner(encoder.d_model, dim, kernel).to(next(encoder.parameters()).device)
def clean_frames(module, args, out):
if isinstance(out, tuple):
return (encoder.fv_insert(out[0]),) + tuple(out[1:])
return encoder.fv_insert(out)
encoder.pre_encode.register_forward_hook(clean_frames)
return encoder.fv_insert
def state_dict_of(nemo_path):
with tarfile.open(nemo_path) as tar, tempfile.TemporaryDirectory() as tmp:
member = next(m for m in tar.getmembers() if os.path.basename(m.name) == "model_weights.ckpt")
tar.extract(member, tmp, filter="data")
return torch.load(os.path.join(tmp, member.name), map_location="cpu", weights_only=True)
def resolve(model_id):
if os.path.exists(model_id):
return model_id
from huggingface_hub import hf_hub_download, list_repo_files
nemo_files = [f for f in list_repo_files(model_id) if f.endswith(".nemo")]
assert len(nemo_files) == 1, f"expected one .nemo file in {model_id}, found {nemo_files}"
return hf_hub_download(model_id, nemo_files[0])
def split_clean_mode(state):
"""Take the adapter's tensors (`clean_mode.<module>.A` / `.B`) out of the checkpoint's state dict."""
adapter = {k[len(CLEAN_MODE_PREFIX):]: state.pop(k) for k in [k for k in state if k.startswith(CLEAN_MODE_PREFIX)]}
return state, adapter
def fold_clean_mode(state, adapter, scale):
"""Add scale * B @ A to every adapted encoder weight, and clean mode's two input vectors to the biases they meet."""
decoder_input, joint_hidden = adapter.pop("decoder_input"), adapter.pop("joint_hidden")
lstm = "decoder.prediction.dec_rnn.lstm"
state[f"{lstm}.bias_ih_l0"] = state[f"{lstm}.bias_ih_l0"] + state[f"{lstm}.weight_ih_l0"] @ decoder_input
state["joint.pred.bias"] = state["joint.pred.bias"] + joint_hidden
for name in sorted({k.rsplit(".", 1)[0] for k in adapter}):
key = f"encoder.{name}.weight"
assert key in state, f"clean mode adapts {key}, which the model does not have"
delta = scale * (adapter[f"{name}.B"] @ adapter[f"{name}.A"])
state[key] = state[key] + delta.view_as(state[key]).to(state[key].dtype)
return state
def load(model_id, map_location=None, clean_mode=False):
path = resolve(model_id)
model = ASRModel.restore_from(path, map_location=map_location, strict=False)
spec = model.cfg.fluid_speech
attach(model, spec.insert_dim, spec.insert_kernel)
state, adapter = split_clean_mode(state_dict_of(path))
if clean_mode:
state = fold_clean_mode(state, adapter, float(spec.clean_mode.alpha) / int(spec.clean_mode.rank))
model.load_state_dict(state, strict=True)
return model.eval()