"""UserLM-2.0 demo: the model plays the user, you play the assistant. Runs on ZeroGPU; also runs locally (CUDA or CPU) because ``@spaces.GPU`` is a no-op outside ZeroGPU. """ from __future__ import annotations import logging import math import os from dataclasses import dataclass import gradio as gr import torch from transformers import ( AutoModelForCausalLM, AutoTokenizer, LogitsProcessorList, RepetitionPenaltyLogitsProcessor, ) # `spaces` must be imported before anything initializes CUDA. import spaces # isort: skip # ====================== # Config # ====================== MODEL_ID = os.getenv("MODEL_ID", "microsoft/UserLM-2.0") MAX_CONTEXT_TOKENS = 8192 # SFT max sequence length reported on the model card DEFAULT_INTENT = ( "You are a user who wants to compute rolling 7-day averages over uneven time stamps. " "You are suspicious of resampling magic and will accuse the assistant of witchcraft " "if it's not explicit." ) EXAMPLE_INTENTS = [ [DEFAULT_INTENT], [ ( "You want help planning a quick vegetarian dinner for four people. You have 45 " "minutes, one guest has a peanut allergy, and you need a shopping list and cooking " "instructions. You prefer pasta, but only mention that preference if the assistant " "asks. Your goal is not satisfied until you receive a menu, shopping list, and " "cooking steps." ) ], [ ( "Create a migration guide for users moving from CanvasDraw to your graphics " "library: outline its structure and main topics, then draft a welcoming " "introduction." ) ], ] # Model-card decoding defaults (generation_config.json ships 0.6 / 0.9; the card uses 1.0 / 0.8). DEFAULT_TEMPERATURE = 1.0 DEFAULT_TOP_P = 0.8 DEFAULT_MAX_NEW_TOKENS = 128 # Not on the model card (which samples without one): a light penalty on the turn being # written curbs within-turn loops ("tell me about X. tell me about Y. ..."). DEFAULT_REPETITION_PENALTY = 1.1 # Guardrail 1 from the UserLM-8B paper (Appendix C.1). Off by default for 2.0, whose # natural lowercase openers ("i need ...") this list would block. FIRST_TOKEN_BLOCKLIST = ["I", "You", "Here", "i", "you", "here"] # ====================== # Load model (module level, on `cuda`, as ZeroGPU expects) # ====================== # On ZeroGPU, `import spaces` makes torch.cuda.is_available() return True at import time. DEVICE = "cuda" if torch.cuda.is_available() else "cpu" tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) # Checkpoint is stored in fp32 (~32 GB); cast to bf16 (~16 GB) on load. model = ( AutoModelForCausalLM.from_pretrained(MODEL_ID, dtype=torch.bfloat16) .to(DEVICE) .eval() ) END_CONV = "<|endconversation|>" END_CONV_ID = tokenizer.convert_tokens_to_ids(END_CONV) if END_CONV_ID is None or END_CONV_ID == tokenizer.unk_token_id: raise RuntimeError( f"{MODEL_ID} has no {END_CONV} token; this demo targets UserLM-2.0." ) _gen_eos = model.generation_config.eos_token_id _gen_eos = [_gen_eos] if isinstance(_gen_eos, int) else list(_gen_eos or []) STOP_IDS = list( dict.fromkeys([*_gen_eos, tokenizer.eos_token_id]) ) # eot / eom / end_of_text PAD_ID = ( tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id ) FIRST_TOKEN_IDS = sorted( { ids[0] for w in FIRST_TOKEN_BLOCKLIST if (ids := tokenizer.encode(w, add_special_tokens=False)) } ) # history_pairs: list of (userlm_turn, your_reply_or_None) Pairs = list[tuple[str, str | None]] # ====================== # Prompt building (CPU, before any GPU is requested) # ====================== def build_messages(intent: str, pairs: Pairs) -> list[dict[str, str]]: msgs = [{"role": "system", "content": intent.strip()}] for userlm_turn, your_reply in pairs: msgs.append({"role": "user", "content": userlm_turn}) if your_reply is not None: msgs.append({"role": "assistant", "content": your_reply}) return msgs def build_prompt_ids(intent: str, pairs: Pairs) -> list[int]: # The bundled chat template opens a `user` header when add_generation_prompt=True. return tokenizer.apply_chat_template( build_messages(intent, pairs), add_generation_prompt=True, tokenize=True, return_dict=True, )["input_ids"] # ====================== # Decoding + guardrails 3 & 4 (pure CPU helpers) # ====================== @dataclass class Candidate: text: str ended: bool def decode_candidate(ids: list[int]) -> Candidate: ended = END_CONV_ID in ids if ended: ids = ids[: ids.index(END_CONV_ID)] # <|endconversation|> is a non-special token: cut it here, skip_special_tokens won't. return Candidate(tokenizer.decode(ids, skip_special_tokens=True).strip(), ended) def normalize(s: str) -> str: return " ".join(s.lower().split()) def passes_guardrails( cand: Candidate, word_window: tuple[int, int] | None, seen_texts: set[str] | None, ) -> bool: """Word-count window and verbatim-repetition filters; `None` disables each.""" if cand.ended and not cand.text: return True # a bare termination signal is always a valid turn if not cand.text: return False if word_window and not word_window[0] <= len(cand.text.split()) <= word_window[1]: return False return not (seen_texts and normalize(cand.text) in seen_texts) # ====================== # Generation (the only GPU code) # ====================== def _sample( prompt_ids: list[int], n: int, max_new_tokens: int, temperature: float, top_p: float, repetition_penalty: float, allow_end: bool, block_first_tokens: bool, ) -> list[list[int]]: input_ids = torch.tensor([prompt_ids], device=model.device) gen_kwargs = { "do_sample": True, "temperature": temperature, "top_p": top_p, "max_new_tokens": max_new_tokens, "num_return_sequences": n, "pad_token_id": PAD_ID, # Stop on <|endconversation|> too, so a terminating turn costs one decode step. "eos_token_id": [*STOP_IDS, END_CONV_ID] if allow_end else STOP_IDS, } if not allow_end: gen_kwargs["suppress_tokens"] = [END_CONV_ID] # guardrail 2 if block_first_tokens and FIRST_TOKEN_IDS: gen_kwargs["begin_suppress_tokens"] = FIRST_TOKEN_IDS # guardrail 1 if repetition_penalty != 1.0: # Penalize only the turn being written. The stock `repetition_penalty` kwarg # also penalizes every prompt token, which here includes <|eot_id|> after each # earlier turn (discouraging the simulated user from ending its turn) and the # intent's own wording (discouraging it from stating its constraints). gen_kwargs["logits_processor"] = LogitsProcessorList( [ RepetitionPenaltyLogitsProcessor( repetition_penalty, prompt_ignore_length=input_ids.shape[1] ) ] ) with torch.inference_mode(): out = model.generate( input_ids=input_ids, attention_mask=torch.ones_like(input_ids), **gen_kwargs ) return out[:, input_ids.shape[1] :].tolist() def estimate_gpu_seconds( prompt_ids: list[int], max_new_tokens: int, temperature: float, top_p: float, repetition_penalty: float, allow_end: bool, block_first_tokens: bool, word_window: tuple[int, int] | None, seen_texts: set[str] | None, retry_batch: int, ) -> int: """Dynamic ZeroGPU duration; must accept the same arguments as `generate_user_turn`. Shorter requests get better queue priority and fit in small remaining quotas. Budget: ~15 s fixed overhead plus ~50 ms per decode step for 8B bf16 with HF `generate` (an estimate, not a measurement), doubled when a retry round can run. """ rounds = 2 if (word_window or seen_texts) and retry_batch > 0 else 1 return math.ceil(min(120, 15 + 0.05 * max_new_tokens * rounds)) @spaces.GPU(duration=estimate_gpu_seconds) def generate_user_turn( prompt_ids: list[int], max_new_tokens: int, temperature: float, top_p: float, repetition_penalty: float, allow_end: bool, block_first_tokens: bool, word_window: tuple[int, int] | None, seen_texts: set[str] | None, retry_batch: int, ) -> tuple[list[int], int, bool]: """Sample one user turn; if it fails a filter, resample `retry_batch` in one batch. Returns (token_ids, rejected_count, passed). Inputs and outputs are plain Python objects so they cross the ZeroGPU process boundary cheaply. """ args = ( max_new_tokens, temperature, top_p, repetition_penalty, allow_end, block_first_tokens, ) first = _sample(prompt_ids, 1, *args)[0] if passes_guardrails(decode_candidate(first), word_window, seen_texts): return first, 0, True if retry_batch < 1: return first, 1, False for i, ids in enumerate(_sample(prompt_ids, retry_batch, *args)): if passes_guardrails(decode_candidate(ids), word_window, seen_texts): return ids, i + 1, True return first, 1 + retry_batch, False # ====================== # Conversation logic (CPU) # ====================== # State: `pairs` is a list of (userlm_turn, your_reply_or_None); `ended` is "" while the # conversation is open, "model" after <|endconversation|>, or "limit" at the context cap. log = logging.getLogger("userlm-demo") END_NOTES = { "model": ( "šŸ The simulated user ended the conversation", ( "It emitted `<|endconversation|>`: the task intent counts as satisfied. " "Retry to resample this turn, or undo to revise your last reply." ), ), "limit": ( "Context limit reached", ( f"UserLM-2.0 was trained on conversations up to {MAX_CONTEXT_TOKENS} tokens. " "Undo your last reply or start a new conversation." ), ), } REPLY_PLACEHOLDER = { "active": "Reply as the assistant…", "waiting": "Waiting for the simulated user…", } START_LABEL = {"idle": "Start conversation", "ended": "Start a new conversation"} def intent_message(intent: str) -> dict: """The transcript opens with the intent on your side: it is what you hand the simulated user, and Gradio's retry/undo need a user-role message before the simulated user's opener (without one, its frontend handler errors out).""" intent = intent.strip() preview = intent if len(intent) <= 48 else intent[:47].rstrip() + "…" return { "role": "user", "content": intent or "*Empty: the simulated user is unconditioned.*", "metadata": {"title": "Task intent", "log": preview, "status": "done"}, } def to_chat_messages(pairs: Pairs, ended: str, intent: str) -> list[dict]: """Left bubbles (role='assistant') = UserLM; right bubbles (role='user') = you.""" ui: list[dict] = [intent_message(intent)] for userlm_turn, your_reply in pairs: if userlm_turn: ui.append({"role": "assistant", "content": userlm_turn}) if your_reply is not None: ui.append({"role": "user", "content": your_reply}) if ended: title, body = END_NOTES[ended] ui.append({"role": "assistant", "content": body, "metadata": {"title": title}}) return ui def message_owners(pairs: Pairs) -> list[tuple[int, str]]: """(pair index, 'intent' | 'userlm' | 'you') for each chat message, in order.""" owners: list[tuple[int, str]] = [(-1, "intent")] for i, (userlm_turn, your_reply) in enumerate(pairs): if userlm_turn: owners.append((i, "userlm")) if your_reply is not None: owners.append((i, "you")) return owners def drop_open_userlm_turn(pairs: Pairs) -> Pairs: return pairs[:-1] if pairs and pairs[-1][1] is None else pairs def render( pairs: Pairs, ended: str, intent: str, status: str = "", reply_value: str = "" ): """Every UI output for a settled state: idle, active, or ended. The composer slot holds one control at a time: the reply box while the conversation is open, otherwise the Start button (so it is never pushed below the fold on phones). """ phase = "ended" if ended else ("active" if pairs else "idle") return ( to_chat_messages(pairs, ended, intent) if phase != "idle" else [], pairs, ended, gr.update( value=reply_value, visible=phase == "active", interactive=True, placeholder=REPLY_PLACEHOLDER["active"], ), gr.update( visible=phase != "active", value=START_LABEL.get(phase, START_LABEL["idle"]), ), status, ) def waiting(pairs: Pairs, intent: str): """UI while the next user turn is being generated; `next_turn` follows.""" return ( to_chat_messages(pairs, "", intent), pairs, "", gr.update( value="", visible=True, interactive=False, placeholder=REPLY_PLACEHOLDER["waiting"], ), gr.update(visible=False), "", ) N_OUTPUTS = len(render([], "", "")) NO_CHANGE = (gr.skip(),) * N_OUTPUTS def _as_text(value) -> str: if isinstance(value, str): return value if isinstance(value, dict): return str(value.get("text") or value.get("content") or "") if isinstance(value, list): return " ".join(_as_text(v) for v in value) return "" def begin(intent: str): """Clear the chat; `next_turn` then writes the opener.""" return waiting([], intent) def add_reply(your_reply: str, pairs: Pairs, ended: str, intent: str): """Show your reply immediately, before the GPU call for the next user turn.""" your_reply = (your_reply or "").strip() if ended or not pairs or pairs[-1][1] is not None or not your_reply: return NO_CHANGE return waiting([*pairs[:-1], (pairs[-1][0], your_reply)], intent) def retry(pairs: Pairs, intent: str): """Resample the simulated user's latest turn (including the opener or an ending).""" return waiting(drop_open_userlm_turn(pairs), intent) def undo(pairs: Pairs, intent: str): """Remove the latest user turn and your reply before it; restore that reply.""" pairs = drop_open_userlm_turn(pairs) if not pairs: return render([], "", intent) *rest, (userlm_turn, your_reply) = pairs return render([*rest, (userlm_turn, None)], "", intent, reply_value=your_reply) def edit(pairs: Pairs, ended: str, intent: str, data: gr.EditData): """Edit any message; the conversation is truncated after it and continues. Returns the usual outputs plus an update for the sidebar intent box. """ owners = message_owners(pairs) index = data.index if isinstance(data.index, int) else data.index[0] new_text = _as_text(data.value).strip() if index >= len(owners) or not new_text: # the end note, or an empty edit return (*render(pairs, ended, intent), gr.skip()) i, who = owners[index] if who == "intent": # same as editing the sidebar: applies from the next turn return (*render(pairs, ended, new_text), new_text) if who == "userlm": # keep the edited user turn; wait for your reply return (*render([*pairs[:i], (new_text, None)], "", intent), gr.skip()) return (*waiting([*pairs[:i], (pairs[i][0], new_text)], intent), gr.skip()) def next_turn( pairs: Pairs, ended: str, intent: str, max_new_tokens: int, temperature: float, top_p: float, repetition_penalty: float, allow_end: bool, block_first_tokens: bool, length_filter: bool, min_words: int, max_words: int, repetition_filter: bool, retry_batch: int, ): if ended or (pairs and pairs[-1][1] is None): return NO_CHANGE # nothing new to respond to intent = (intent or "").strip() # read live from the textbox on every turn prompt_ids = build_prompt_ids(intent, pairs) if len(prompt_ids) + max_new_tokens > MAX_CONTEXT_TOKENS: return render(pairs, "limit", intent) word_window = (int(min_words), int(max_words)) if length_filter else None seen_texts = ( {normalize(intent)} | {normalize(u) for u, _ in pairs} if repetition_filter else None ) try: ids, rejected, passed = generate_user_turn( prompt_ids, int(max_new_tokens), float(temperature), float(top_p), float(repetition_penalty), bool(allow_end), bool(block_first_tokens), word_window, seen_texts, int(retry_batch), ) except Exception as e: # e.g. ZeroGPU quota or timeout: roll back, keep the text log.exception("generation failed") gr.Warning(f"Generation failed: {e}") if not pairs: return render([], "", intent) *rest, (last_user, last_reply) = pairs return render([*rest, (last_user, None)], "", intent, reply_value=last_reply) chosen = decode_candidate(ids) status = "" if not passed: status = ( f"All {rejected} sampled turns failed the guardrails, so this one is " "shown unfiltered." ) elif rejected: status = f"Guardrails rejected {rejected} sampled turn(s) before this one." if chosen.ended: if chosen.text: pairs = [*pairs, (chosen.text, None)] return render(pairs, "model", intent, status) return render([*pairs, (chosen.text, None)], "", intent, status) def reset(): return render([], "", "") # ====================== # Gradio UI # ====================== THEME = gr.themes.Base( primary_hue=gr.themes.colors.teal, neutral_hue=gr.themes.colors.slate, font=[gr.themes.GoogleFont("IBM Plex Sans"), "ui-sans-serif", "system-ui"], font_mono=[gr.themes.GoogleFont("IBM Plex Mono"), "ui-monospace", "monospace"], ) CSS = """ :root { --sim-bg: #f3e8d6; --sim-border: #e2cfae; --you-bg: #e3f1ee; --you-border: #b3d8d0; --end-bg: #fbf0dc; --end-border: #e0ad5a; --end-text: #6b430b; } .dark { --sim-bg: #362e23; --sim-border: #52463a; --you-bg: #12312d; --you-border: #24564e; --end-bg: #33260f; --end-border: #9c6c1f; --end-text: #f1d29b; } #header h1 { font-size: 1.45rem; font-weight: 600; margin: 0 0 .2rem; } #header p { margin: 0; max-width: 75ch; color: var(--body-text-color-subdued); } @media (max-width: 640px) { #header p { font-size: var(--text-sm); } } #chat .message.bot { background: var(--sim-bg); border-color: var(--sim-border); } #chat .message.user { background: var(--you-bg); border-color: var(--you-border); } #chat .message-row.bubble .message { max-width: min(76ch, 82%); } #chat .message.bot:has(.thought-group) { background: var(--end-bg); border-color: var(--end-border); } #chat .thought-group { background: transparent; border: 0; padding: 0; } #chat .message.bot .thought-group .md, #chat .message.bot .thought-group p { color: var(--end-text); } /* The intent card on your side: outlined, not filled, so it reads as setup. */ #chat .message.user:has(.thought-group) { background: transparent; border-style: dashed; } /* Spell out Retry and Undo: icon-only buttons are ambiguous on touch screens. */ #chat .icon-button[aria-label="Retry"], #chat .icon-button[aria-label="Undo"] { width: auto !important; height: auto !important; white-space: nowrap; gap: .3rem; padding: 2px .5rem !important; font-size: var(--text-sm); line-height: 1.3; } #chat .icon-button[aria-label="Retry"]::after, #chat .icon-button[aria-label="Undo"]::after { position: static; width: auto; height: auto; inset: auto; transform: none; background: none; border: 0; opacity: 1; } #chat .icon-button[aria-label="Retry"]::after { content: "Retry"; } #chat .icon-button[aria-label="Undo"]::after { content: "Undo"; } #status p { margin: 0; font-size: var(--text-sm); color: var(--body-text-color-subdued); } /* Keep the conversation column readable on wide screens. */ #header, #chat, #status, #composer { width: 100%; max-width: 1120px; margin-inline: auto; } """ # Phones: start with the sidebar collapsed so the chat is what loads first. JS = """ () => { if (!window.matchMedia("(max-width: 768px)").matches) return; let tries = 0; const close = () => { const t = document.querySelector('.sidebar.open button[aria-label="Toggle Sidebar"]'); if (t) t.click(); else if (tries++ < 50) setTimeout(close, 100); }; close(); } """ with gr.Blocks(title="UserLM-2.0", fill_height=True) as demo: pairs_state = gr.State([]) ended_state = gr.State("") with gr.Sidebar(width=380): intent_box = gr.Textbox( label="Task intent", info=( "The simulated user's goal, constraints, and what counts as done. " "Read on every turn: edits apply from the next turn, and Retry " "resamples the latest turn with the new intent." ), value=DEFAULT_INTENT, lines=6, max_lines=14, ) gr.Examples( EXAMPLE_INTENTS, inputs=[intent_box], label="Examples", example_labels=["Rolling averages", "Vegetarian dinner", "Migration guide"], ) with gr.Accordion("Sampling", open=False): max_new_tokens = gr.Slider( 16, 512, value=DEFAULT_MAX_NEW_TOKENS, step=16, label="Max new tokens" ) temperature = gr.Slider( 0.1, 2.0, value=DEFAULT_TEMPERATURE, step=0.05, label="Temperature" ) top_p = gr.Slider(0.05, 1.0, value=DEFAULT_TOP_P, step=0.01, label="Top-p") repetition_penalty = gr.Slider( 1.0, 1.5, value=DEFAULT_REPETITION_PENALTY, step=0.01, label="Repetition penalty", info=( "Applies only to the turn being written. " "1.0 is plain sampling, as on the model card." ), ) with gr.Accordion("Guardrails", open=False): gr.Markdown( "From the [UserLM-8B paper](https://arxiv.org/abs/2510.06552), App. C.1. " "Off by default except repetition: the length window and first-token " "filter were tuned for UserLM-8B and reject normal 2.0 turns." ) allow_end = gr.Checkbox( value=True, label="Let the simulated user end the conversation" ) repetition_filter = gr.Checkbox( value=True, label="Reject repeats of the intent or earlier user turns" ) block_first_tokens = gr.Checkbox( value=False, label="Block first tokens: " + ", ".join(FIRST_TOKEN_BLOCKLIST), ) length_filter = gr.Checkbox( value=False, label="Enforce a word-count window" ) with gr.Row(): min_words = gr.Slider(1, 50, value=3, step=1, label="Min words") max_words = gr.Slider(5, 200, value=25, step=1, label="Max words") retry_batch = gr.Slider( 0, 8, value=4, step=1, label="Replacements sampled when a turn fails" ) with gr.Accordion("How this demo works", open=False): gr.Markdown( f""" The intent is the system message. UserLM's turns are `user` messages and yours are `assistant` messages, and the model's chat template opens the next `user` turn. Sampling follows the [model card](https://huggingface.co/{MODEL_ID}) (temperature 1.0, top-p 0.8, 128 new tokens), plus a light repetition penalty (1.1) on the turn being written to curb loops. Generation stops at `<|eot_id|>` or `<|endconversation|>`. If a sampled turn fails a guardrail, replacements are sampled as one batch inside the same GPU call and the first passing one is kept. On ZeroGPU each turn is a single `@spaces.GPU` call whose time budget is computed from the max new tokens. """ ) gr.Markdown( f""" # UserLM-2.0 You play the assistant. [UserLM-2.0](https://huggingface.co/{MODEL_ID}) plays a user pursuing the task intent in the sidebar, and ends the chat once it is satisfied. """, elem_id="header", ) chatbot = gr.Chatbot( elem_id="chat", show_label=False, scale=1, # Base height shrinks on short screens so the composer stays on screen; # fill_height still grows it to the window on larger ones. height="clamp(200px, calc(100dvh - 320px), 400px)", autoscroll=True, buttons=["copy", "copy_all"], editable="all", group_consecutive_messages=False, placeholder=( "**Press Start conversation** to get the simulated user's opening message." "
Its messages appear on the left; your replies as the assistant appear " "on the right." ), ) status_md = gr.Markdown(elem_id="status") with gr.Row(equal_height=True, elem_id="composer"): reply_box = gr.Textbox( show_label=False, placeholder=REPLY_PLACEHOLDER["active"], visible=False, lines=1, max_lines=8, submit_btn=True, scale=1, ) start_btn = gr.Button(START_LABEL["idle"], variant="primary", scale=1) settings = [ max_new_tokens, temperature, top_p, repetition_penalty, allow_end, block_first_tokens, length_filter, min_words, max_words, repetition_filter, retry_batch, ] outputs = [ chatbot, pairs_state, ended_state, reply_box, start_btn, status_md, ] turn = { "fn": next_turn, "inputs": [pairs_state, ended_state, intent_box, *settings], "outputs": outputs, "show_progress_on": [chatbot], "api_visibility": "private", } private = {"api_visibility": "private"} pending_turns = [ start_btn.click(begin, [intent_box], outputs, queue=False, **private).then( **turn ), reply_box.submit( add_reply, [reply_box, pairs_state, ended_state, intent_box], outputs, queue=False, **private, ).then(**turn), chatbot.retry( retry, [pairs_state, intent_box], outputs, queue=False, **private ).then(**turn), chatbot.edit( edit, [pairs_state, ended_state, intent_box], [*outputs, intent_box], queue=False, **private, ).then(**turn), ] chatbot.undo(undo, [pairs_state, intent_box], outputs, queue=False, **private) # The Chatbot's own trash icon resets the whole conversation. chatbot.clear(reset, None, outputs, **private) # Editing, undoing, retrying or clearing while a turn is still generating # cancels it, so a stale result can't overwrite the new conversation state. for trigger in (chatbot.edit, chatbot.undo, chatbot.retry, chatbot.clear): trigger(None, None, None, cancels=pending_turns, queue=False, **private) if __name__ == "__main__": demo.launch(theme=THEME, css=CSS, js=JS)