Xenova HF Staff commited on
Commit
824a5b6
·
verified ·
1 Parent(s): 5f13060

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +20 -22
README.md CHANGED
@@ -135,19 +135,11 @@ from huggingface_hub import snapshot_download
135
  model_id = "onnx-community/LFM2-24B-A2B-ONNX"
136
  config = AutoConfig.from_pretrained(model_id)
137
  tokenizer = AutoTokenizer.from_pretrained(model_id)
138
-
139
- local_dir = "LFM2-24B-A2B-ONNX"
140
- model_path = snapshot_download(model_id, local_dir=local_dir, allow_patterns=["onnx/model_q4f16.onnx", "onnx/model_q4f16.onnx_data*"]) # Download model graph and weights
141
- session = onnxruntime.InferenceSession(model_path)
142
-
143
- ## Set config values
144
- num_key_value_heads = config.num_key_value_heads
145
- head_dim = config.hidden_size // config.num_attention_heads
146
- num_hidden_layers = config.num_hidden_layers
147
  eos_token_id = config.eos_token_id
148
- hidden_size = config.hidden_size
149
- conv_L_cache = config.conv_L_cache
150
- layer_types = config.layer_types
 
151
 
152
  # 2. Prepare inputs
153
  prompt = "What is C. elegans?"
@@ -156,15 +148,19 @@ inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tok
156
  input_ids = inputs['input_ids']
157
  attention_mask = inputs['attention_mask']
158
  batch_size = input_ids.shape[0]
 
 
159
  past_cache_values = {}
160
- for i in range(num_hidden_layers):
161
- if layer_types[i] == 'full_attention':
162
- for kv in ('key', 'value'):
163
- past_cache_values[f'past_key_values.{i}.{kv}'] = np.zeros([batch_size, num_key_value_heads, 0, head_dim], dtype=np.float32)
164
- elif layer_types[i] == 'conv':
165
- past_cache_values[f'past_conv.{i}'] = np.zeros([batch_size, hidden_size, conv_L_cache], dtype=np.float32)
166
- else:
167
- raise ValueError(f"Unsupported layer type: {layer_types[i]}")
 
 
168
 
169
  # 3. Generation loop
170
  max_new_tokens = 1024
@@ -173,6 +169,7 @@ for i in range(max_new_tokens):
173
  logits, *present_cache_values = session.run(None, dict(
174
  input_ids=input_ids,
175
  attention_mask=attention_mask,
 
176
  **past_cache_values,
177
  ))
178
 
@@ -182,8 +179,8 @@ for i in range(max_new_tokens):
182
  for j, key in enumerate(past_cache_values):
183
  past_cache_values[key] = present_cache_values[j]
184
  generated_tokens = np.concatenate([generated_tokens, input_ids], axis=-1)
185
- if (input_ids == eos_token_id).all():
186
- break
187
 
188
  ## (Optional) Streaming
189
  print(tokenizer.decode(input_ids[0]), end='', flush=True)
@@ -191,6 +188,7 @@ print()
191
 
192
  # 4. Output result
193
  print(tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)[0])
 
194
  ```
195
 
196
  ## 🔧 Fine-Tuning
 
135
  model_id = "onnx-community/LFM2-24B-A2B-ONNX"
136
  config = AutoConfig.from_pretrained(model_id)
137
  tokenizer = AutoTokenizer.from_pretrained(model_id)
 
 
 
 
 
 
 
 
 
138
  eos_token_id = config.eos_token_id
139
+
140
+ filename = "model_q4f16.onnx" # Options: "model_fp16.onnx", "model_q4f16.onnx"
141
+ model_path = snapshot_download(repo_id=model_id, allow_patterns=f"onnx/{filename}*") # Download the graph + weights
142
+ session = onnxruntime.InferenceSession(f"{model_path}/onnx/{filename}")
143
 
144
  # 2. Prepare inputs
145
  prompt = "What is C. elegans?"
 
148
  input_ids = inputs['input_ids']
149
  attention_mask = inputs['attention_mask']
150
  batch_size = input_ids.shape[0]
151
+ num_logits_to_keep = np.array(1, dtype=np.int64)
152
+
153
  past_cache_values = {}
154
+ for inp in session.get_inputs():
155
+ name = inp.name
156
+ shape = inp.shape
157
+ dtype = np.float32 if inp.type == "tensor(float)" else np.float16
158
+ if name.startswith("past_key_values"):
159
+ # Attention KV cache: shape [batch_size, num_kv_heads, 0, head_dim]
160
+ past_cache_values[name] = np.zeros([batch_size, shape[1], 0, shape[3]], dtype=dtype)
161
+ elif name.startswith("past_conv"):
162
+ # Conv cache: shape [batch_size, hidden_size, conv_L_cache]
163
+ past_cache_values[name] = np.zeros([batch_size, shape[1], shape[2]], dtype=dtype)
164
 
165
  # 3. Generation loop
166
  max_new_tokens = 1024
 
169
  logits, *present_cache_values = session.run(None, dict(
170
  input_ids=input_ids,
171
  attention_mask=attention_mask,
172
+ num_logits_to_keep=num_logits_to_keep,
173
  **past_cache_values,
174
  ))
175
 
 
179
  for j, key in enumerate(past_cache_values):
180
  past_cache_values[key] = present_cache_values[j]
181
  generated_tokens = np.concatenate([generated_tokens, input_ids], axis=-1)
182
+ if np.isin(input_ids, eos_token_id).any():
183
+ break
184
 
185
  ## (Optional) Streaming
186
  print(tokenizer.decode(input_ids[0]), end='', flush=True)
 
188
 
189
  # 4. Output result
190
  print(tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)[0])
191
+
192
  ```
193
 
194
  ## 🔧 Fine-Tuning