mmardani commited on
Commit
5cd3aa4
·
verified ·
1 Parent(s): 221782c

fix: declare projected_bernstein schedule parameterization

Browse files
config.json CHANGED
@@ -52,5 +52,7 @@
52
  "pad_token_id": null,
53
  "bos_token_id": 1,
54
  "eos_token_id": 2,
55
- "torch_dtype": "bfloat16"
 
 
56
  }
 
52
  "pad_token_id": null,
53
  "bos_token_id": 1,
54
  "eos_token_id": 2,
55
+ "torch_dtype": "bfloat16",
56
+ "schedule_parameterization": "projected_bernstein",
57
+ "schedule_degree": 32
58
  }
configuration_nemotron_labs_sigma.py CHANGED
@@ -78,6 +78,8 @@ class NemotronLabsSigmaConfig(_PretrainedConfig):
78
  self.rope_parameters = rope_parameters
79
  # transformers 4.x reads YaRN settings from `rope_scaling`.
80
  self.rope_scaling = dict(rope_parameters)
 
 
81
  self.embed_dim = embed_dim
82
  self.block_size = block_size
83
  self.replaid_selfcond = replaid_selfcond
 
78
  self.rope_parameters = rope_parameters
79
  # transformers 4.x reads YaRN settings from `rope_scaling`.
80
  self.rope_scaling = dict(rope_parameters)
81
+ self.schedule_parameterization = kwargs.pop("schedule_parameterization", "legacy")
82
+ self.schedule_degree = kwargs.pop("schedule_degree", 32)
83
  self.embed_dim = embed_dim
84
  self.block_size = block_size
85
  self.replaid_selfcond = replaid_selfcond
modeling_nemotron_labs_sigma.py CHANGED
@@ -238,9 +238,49 @@ class NLCDEncoder(nn.Module):
238
  # ---------------------------------------------------------------------------
239
 
240
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
241
  class NLCDNoiseSchedule(nn.Module):
242
- """Learned monotone map t -> gamma_tilde in [0, 1] (kept for checkpoint
243
- compatibility; the default edm_sigma sampler does not evaluate it)."""
 
244
 
245
  def __init__(self):
246
  super().__init__()
@@ -263,6 +303,8 @@ class NLCDNoiseSchedule(nn.Module):
263
 
264
 
265
  class NLCDGammaBounds(nn.Module):
 
 
266
  def __init__(self, gamma_0=-3.0, gamma_1=6.0):
267
  super().__init__()
268
  self.gamma_0 = nn.Parameter(torch.tensor(float(gamma_0)))
@@ -272,6 +314,22 @@ class NLCDGammaBounds(nn.Module):
272
  return self.gamma_0.double(), self.gamma_1.double()
273
 
274
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
275
  class NLCDEmbeddingMatrix(nn.Module):
276
  """Vocabulary embedding matrix E in R^{V x embed_dim}, rows on the unit sphere."""
277
 
@@ -307,8 +365,17 @@ class NemotronLabsSigmaModel(PreTrainedModel):
307
  super().__init__(config)
308
  self.encoder = NLCDEncoder(config)
309
  self.diffusion_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
310
- self.noise_schedule = NLCDNoiseSchedule()
311
- self.gamma_bounds = NLCDGammaBounds()
 
 
 
 
 
 
 
 
 
312
  self.replaid_embedding_matrix = NLCDEmbeddingMatrix(config.vocab_size, config.embed_dim)
313
  self.replaid_input_linear = nn.Linear(config.embed_dim, config.hidden_size, bias=False)
314
  if config.replaid_selfcond:
 
238
  # ---------------------------------------------------------------------------
239
 
240
 
241
+ class NLCDBernsteinNoiseSchedule(nn.Module):
242
+ """Monotone degree-N Bernstein (Bezier) map t -> gamma_tilde in [0, 1].
243
+
244
+ This is the parameterization used by the MS-SUSReg lineage: the free
245
+ parameter is the vector of simplex increments, whose cumulative sum gives
246
+ the Bezier control points. Uniform increments reproduce s(t) = t.
247
+ """
248
+
249
+ def __init__(self, degree=32):
250
+ super().__init__()
251
+ self.degree = int(degree)
252
+ self.increment_values = nn.Parameter(
253
+ torch.full((self.degree,), 1.0 / self.degree)
254
+ )
255
+
256
+ def increments(self):
257
+ return self.increment_values.double()
258
+
259
+ def control_points(self):
260
+ inc = self.increments()
261
+ return torch.cat((inc.new_zeros(1), inc.cumsum(dim=0)))
262
+
263
+ @staticmethod
264
+ def _de_casteljau(t, control_points):
265
+ t = t.double()
266
+ values = control_points.double().expand(t.numel(), -1)
267
+ flat_t = t.reshape(-1, 1)
268
+ one_minus_t = 1.0 - flat_t
269
+ for remaining in range(control_points.numel() - 1, 0, -1):
270
+ values = (
271
+ one_minus_t * values[:, :remaining]
272
+ + flat_t * values[:, 1 : remaining + 1]
273
+ )
274
+ return values[:, 0].reshape(t.shape)
275
+
276
+ def forward(self, t):
277
+ return self._de_casteljau(t, self.control_points())
278
+
279
+
280
  class NLCDNoiseSchedule(nn.Module):
281
+ """Legacy learned monotone map t -> gamma_tilde in [0, 1] (kept for
282
+ checkpoint compatibility; the default edm_sigma sampler does not
283
+ evaluate it)."""
284
 
285
  def __init__(self):
286
  super().__init__()
 
303
 
304
 
305
  class NLCDGammaBounds(nn.Module):
306
+ """Legacy gamma endpoints stored as two scalar parameters."""
307
+
308
  def __init__(self, gamma_0=-3.0, gamma_1=6.0):
309
  super().__init__()
310
  self.gamma_0 = nn.Parameter(torch.tensor(float(gamma_0)))
 
314
  return self.gamma_0.double(), self.gamma_1.double()
315
 
316
 
317
+ class NLCDProjectedGammaBounds(nn.Module):
318
+ """Gamma endpoints stored as a single 2-vector (MS-SUSReg lineage)."""
319
+
320
+ def __init__(self, gamma_0=-3.0, gamma_1=6.0):
321
+ super().__init__()
322
+ self.endpoint_values = nn.Parameter(
323
+ torch.tensor([float(gamma_0), float(gamma_1)])
324
+ )
325
+
326
+ def forward(self):
327
+ return (
328
+ self.endpoint_values[0].double(),
329
+ self.endpoint_values[1].double(),
330
+ )
331
+
332
+
333
  class NLCDEmbeddingMatrix(nn.Module):
334
  """Vocabulary embedding matrix E in R^{V x embed_dim}, rows on the unit sphere."""
335
 
 
365
  super().__init__(config)
366
  self.encoder = NLCDEncoder(config)
367
  self.diffusion_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
368
+ # Two schedule parameterizations exist in the wild. The MS-SUSReg
369
+ # lineage stores Bernstein increments and a gamma endpoint 2-vector;
370
+ # earlier checkpoints store an MLP schedule and two gamma scalars.
371
+ if getattr(config, "schedule_parameterization", "legacy") == "projected_bernstein":
372
+ self.noise_schedule = NLCDBernsteinNoiseSchedule(
373
+ degree=getattr(config, "schedule_degree", 32)
374
+ )
375
+ self.gamma_bounds = NLCDProjectedGammaBounds()
376
+ else:
377
+ self.noise_schedule = NLCDNoiseSchedule()
378
+ self.gamma_bounds = NLCDGammaBounds()
379
  self.replaid_embedding_matrix = NLCDEmbeddingMatrix(config.vocab_size, config.embed_dim)
380
  self.replaid_input_linear = nn.Linear(config.embed_dim, config.hidden_size, bias=False)
381
  if config.replaid_selfcond: