Instructions to use onnx-community/LFM2-24B-A2B-ONNX with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers.js
How to use onnx-community/LFM2-24B-A2B-ONNX with Transformers.js:
// npm i @huggingface/transformers import { pipeline } from '@huggingface/transformers'; // Allocate pipeline const pipe = await pipeline('text-generation', 'onnx-community/LFM2-24B-A2B-ONNX');
Update README.md
Browse files
README.md
CHANGED
|
@@ -135,19 +135,11 @@ from huggingface_hub import snapshot_download
|
|
| 135 |
model_id = "onnx-community/LFM2-24B-A2B-ONNX"
|
| 136 |
config = AutoConfig.from_pretrained(model_id)
|
| 137 |
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
| 138 |
-
|
| 139 |
-
local_dir = "LFM2-24B-A2B-ONNX"
|
| 140 |
-
model_path = snapshot_download(model_id, local_dir=local_dir, allow_patterns=["onnx/model_q4f16.onnx", "onnx/model_q4f16.onnx_data*"]) # Download model graph and weights
|
| 141 |
-
session = onnxruntime.InferenceSession(model_path)
|
| 142 |
-
|
| 143 |
-
## Set config values
|
| 144 |
-
num_key_value_heads = config.num_key_value_heads
|
| 145 |
-
head_dim = config.hidden_size // config.num_attention_heads
|
| 146 |
-
num_hidden_layers = config.num_hidden_layers
|
| 147 |
eos_token_id = config.eos_token_id
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
|
|
|
| 151 |
|
| 152 |
# 2. Prepare inputs
|
| 153 |
prompt = "What is C. elegans?"
|
|
@@ -156,15 +148,19 @@ inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tok
|
|
| 156 |
input_ids = inputs['input_ids']
|
| 157 |
attention_mask = inputs['attention_mask']
|
| 158 |
batch_size = input_ids.shape[0]
|
|
|
|
|
|
|
| 159 |
past_cache_values = {}
|
| 160 |
-
for
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
|
|
|
|
|
|
| 168 |
|
| 169 |
# 3. Generation loop
|
| 170 |
max_new_tokens = 1024
|
|
@@ -173,6 +169,7 @@ for i in range(max_new_tokens):
|
|
| 173 |
logits, *present_cache_values = session.run(None, dict(
|
| 174 |
input_ids=input_ids,
|
| 175 |
attention_mask=attention_mask,
|
|
|
|
| 176 |
**past_cache_values,
|
| 177 |
))
|
| 178 |
|
|
@@ -182,8 +179,8 @@ for i in range(max_new_tokens):
|
|
| 182 |
for j, key in enumerate(past_cache_values):
|
| 183 |
past_cache_values[key] = present_cache_values[j]
|
| 184 |
generated_tokens = np.concatenate([generated_tokens, input_ids], axis=-1)
|
| 185 |
-
if (input_ids
|
| 186 |
-
|
| 187 |
|
| 188 |
## (Optional) Streaming
|
| 189 |
print(tokenizer.decode(input_ids[0]), end='', flush=True)
|
|
@@ -191,6 +188,7 @@ print()
|
|
| 191 |
|
| 192 |
# 4. Output result
|
| 193 |
print(tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)[0])
|
|
|
|
| 194 |
```
|
| 195 |
|
| 196 |
## 🔧 Fine-Tuning
|
|
|
|
| 135 |
model_id = "onnx-community/LFM2-24B-A2B-ONNX"
|
| 136 |
config = AutoConfig.from_pretrained(model_id)
|
| 137 |
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 138 |
eos_token_id = config.eos_token_id
|
| 139 |
+
|
| 140 |
+
filename = "model_q4f16.onnx" # Options: "model_fp16.onnx", "model_q4f16.onnx"
|
| 141 |
+
model_path = snapshot_download(repo_id=model_id, allow_patterns=f"onnx/{filename}*") # Download the graph + weights
|
| 142 |
+
session = onnxruntime.InferenceSession(f"{model_path}/onnx/{filename}")
|
| 143 |
|
| 144 |
# 2. Prepare inputs
|
| 145 |
prompt = "What is C. elegans?"
|
|
|
|
| 148 |
input_ids = inputs['input_ids']
|
| 149 |
attention_mask = inputs['attention_mask']
|
| 150 |
batch_size = input_ids.shape[0]
|
| 151 |
+
num_logits_to_keep = np.array(1, dtype=np.int64)
|
| 152 |
+
|
| 153 |
past_cache_values = {}
|
| 154 |
+
for inp in session.get_inputs():
|
| 155 |
+
name = inp.name
|
| 156 |
+
shape = inp.shape
|
| 157 |
+
dtype = np.float32 if inp.type == "tensor(float)" else np.float16
|
| 158 |
+
if name.startswith("past_key_values"):
|
| 159 |
+
# Attention KV cache: shape [batch_size, num_kv_heads, 0, head_dim]
|
| 160 |
+
past_cache_values[name] = np.zeros([batch_size, shape[1], 0, shape[3]], dtype=dtype)
|
| 161 |
+
elif name.startswith("past_conv"):
|
| 162 |
+
# Conv cache: shape [batch_size, hidden_size, conv_L_cache]
|
| 163 |
+
past_cache_values[name] = np.zeros([batch_size, shape[1], shape[2]], dtype=dtype)
|
| 164 |
|
| 165 |
# 3. Generation loop
|
| 166 |
max_new_tokens = 1024
|
|
|
|
| 169 |
logits, *present_cache_values = session.run(None, dict(
|
| 170 |
input_ids=input_ids,
|
| 171 |
attention_mask=attention_mask,
|
| 172 |
+
num_logits_to_keep=num_logits_to_keep,
|
| 173 |
**past_cache_values,
|
| 174 |
))
|
| 175 |
|
|
|
|
| 179 |
for j, key in enumerate(past_cache_values):
|
| 180 |
past_cache_values[key] = present_cache_values[j]
|
| 181 |
generated_tokens = np.concatenate([generated_tokens, input_ids], axis=-1)
|
| 182 |
+
if np.isin(input_ids, eos_token_id).any():
|
| 183 |
+
break
|
| 184 |
|
| 185 |
## (Optional) Streaming
|
| 186 |
print(tokenizer.decode(input_ids[0]), end='', flush=True)
|
|
|
|
| 188 |
|
| 189 |
# 4. Output result
|
| 190 |
print(tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)[0])
|
| 191 |
+
|
| 192 |
```
|
| 193 |
|
| 194 |
## 🔧 Fine-Tuning
|