File size: 4,675 Bytes
8104e7e
 
 
 
 
53f490b
412ef0d
36481f6
 
8104e7e
 
 
 
 
 
 
 
 
 
 
 
 
412ef0d
8104e7e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
412ef0d
8104e7e
412ef0d
8104e7e
412ef0d
 
 
 
 
 
 
 
 
 
 
 
36481f6
 
 
 
 
412ef0d
 
 
 
 
8104e7e
 
 
 
 
 
 
 
412ef0d
8104e7e
412ef0d
8104e7e
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
"""Load Fluid Speech: NVIDIA Parakeet TDT 0.6B v3 with one small extra block between subsampling and the first
Conformer layer. The .nemo file holds that block's weights next to the usual ones, so a plain
`ASRModel.from_pretrained` / `restore_from` stops with "Unexpected key(s) ... encoder.fv_insert" instead of quietly
running without it; `load()` below adds the block and then loads every tensor strictly.

Clean mode is an optional transcription style for edited instead of verbatim text (the style of SPGISpeech's
references): it removes repeated words and filler words. Its checkpoint is merged into the model checkpoint, so
every dataset loads the same file; `load(..., clean_mode=True)` switches it on by adding its weights into the model's
before transcribing.

    import fluid_speech
    model = fluid_speech.load("<org>/<model>")                    # Hub repo id, or a local .nemo path
    model = fluid_speech.load("<org>/<model>", clean_mode=True)   # edited, non-verbatim text
"""
import os
import tarfile
import tempfile

import torch
import torch.nn as nn
from nemo.collections.asr.models import ASRModel

CLEAN_MODE_PREFIX = "clean_mode."


class FrameCleaner(nn.Module):
    """Residual block on the subsampled frames: LayerNorm, projection, depthwise time convolution, GELU, projection."""

    def __init__(self, width=1024, dim=1024, kernel=15):
        super().__init__()
        self.ln = nn.LayerNorm(width)
        self.down = nn.Linear(width, dim)
        self.mix = nn.Conv1d(dim, dim, kernel, padding=kernel // 2, groups=dim)
        self.up = nn.Linear(dim, width)

    def forward(self, x):  # x: [batch, time, width]
        h = self.down(self.ln(x)).transpose(1, 2)
        return x + self.up(nn.functional.gelu(self.mix(h)).transpose(1, 2))


def attach(model, dim, kernel):
    """Add the block to `model.encoder` and run it on the output of the subsampling module."""
    encoder = model.encoder
    encoder.fv_insert = FrameCleaner(encoder.d_model, dim, kernel).to(next(encoder.parameters()).device)

    def clean_frames(module, args, out):
        if isinstance(out, tuple):
            return (encoder.fv_insert(out[0]),) + tuple(out[1:])
        return encoder.fv_insert(out)

    encoder.pre_encode.register_forward_hook(clean_frames)
    return encoder.fv_insert


def state_dict_of(nemo_path):
    with tarfile.open(nemo_path) as tar, tempfile.TemporaryDirectory() as tmp:
        member = next(m for m in tar.getmembers() if os.path.basename(m.name) == "model_weights.ckpt")
        tar.extract(member, tmp, filter="data")
        return torch.load(os.path.join(tmp, member.name), map_location="cpu", weights_only=True)


def resolve(model_id):
    if os.path.exists(model_id):
        return model_id
    from huggingface_hub import hf_hub_download, list_repo_files
    nemo_files = [f for f in list_repo_files(model_id) if f.endswith(".nemo")]
    assert len(nemo_files) == 1, f"expected one .nemo file in {model_id}, found {nemo_files}"
    return hf_hub_download(model_id, nemo_files[0])


def split_clean_mode(state):
    """Take the adapter's tensors (`clean_mode.<module>.A` / `.B`) out of the checkpoint's state dict."""
    adapter = {k[len(CLEAN_MODE_PREFIX):]: state.pop(k) for k in [k for k in state if k.startswith(CLEAN_MODE_PREFIX)]}
    return state, adapter


def fold_clean_mode(state, adapter, scale):
    """Add scale * B @ A to every adapted encoder weight, and clean mode's two input vectors to the biases they meet."""
    decoder_input, joint_hidden = adapter.pop("decoder_input"), adapter.pop("joint_hidden")
    lstm = "decoder.prediction.dec_rnn.lstm"
    state[f"{lstm}.bias_ih_l0"] = state[f"{lstm}.bias_ih_l0"] + state[f"{lstm}.weight_ih_l0"] @ decoder_input
    state["joint.pred.bias"] = state["joint.pred.bias"] + joint_hidden
    for name in sorted({k.rsplit(".", 1)[0] for k in adapter}):
        key = f"encoder.{name}.weight"
        assert key in state, f"clean mode adapts {key}, which the model does not have"
        delta = scale * (adapter[f"{name}.B"] @ adapter[f"{name}.A"])
        state[key] = state[key] + delta.view_as(state[key]).to(state[key].dtype)
    return state


def load(model_id, map_location=None, clean_mode=False):
    path = resolve(model_id)
    model = ASRModel.restore_from(path, map_location=map_location, strict=False)
    spec = model.cfg.fluid_speech
    attach(model, spec.insert_dim, spec.insert_kernel)
    state, adapter = split_clean_mode(state_dict_of(path))
    if clean_mode:
        state = fold_clean_mode(state, adapter, float(spec.clean_mode.alpha) / int(spec.clean_mode.rank))
    model.load_state_dict(state, strict=True)
    return model.eval()