{
"cells": [
{
"cell_type": "markdown",
"id": "5a611684",
"metadata": {
"id": "5a611684"
},
"source": [
"# NanoChat Easy - GRPO Training\n",
"\n"
]
},
{
"cell_type": "markdown",
"id": "80df0403",
"metadata": {
"id": "80df0403"
},
"source": [
"## Import model and tokenizer\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1dd76bde",
"metadata": {
"id": "1dd76bde",
"outputId": "b786d7ad-5aa8-4a13-eb1f-54a65aaf44ba"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/fsx/benjamin_burtenshaw/nanochat_/.venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n",
"`torch_dtype` is deprecated! Use `dtype` instead!\n"
]
}
],
"source": [
"import torch\n",
"from torch.utils.data import DataLoader\n",
"from datasets import load_dataset\n",
"from transformers import AutoModelForCausalLM, AutoTokenizer, get_linear_schedule_with_warmup\n",
"\n",
"\n",
"model_id = \"karpathy/nanochat-d32\"\n",
"revision = \"refs/pr/1\"\n",
"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
"\n",
"\n",
"tokenizer = AutoTokenizer.from_pretrained(model_id, revision=revision)\n",
"model = AutoModelForCausalLM.from_pretrained(\n",
" model_id,\n",
" revision=revision,\n",
" torch_dtype=torch.bfloat16 if device.type == \"cuda\" else torch.float32,\n",
").to(device)\n",
"tokenizer.pad_token = tokenizer.eos_token\n",
"model.config.pad_token_id = tokenizer.pad_token_id"
]
},
{
"cell_type": "markdown",
"id": "6eb979a9",
"metadata": {
"id": "6eb979a9"
},
"source": [
"## Setup LoRA\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1973b450",
"metadata": {
"id": "1973b450",
"outputId": "354ceafb-b4cb-4423-f076-7800024171b7"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"trainable params: 1,179,648 || all params: 1,880,227,840 || trainable%: 0.0627\n"
]
}
],
"source": [
"from peft import LoraConfig, get_peft_model\n",
"\n",
"lora_config = LoraConfig(\n",
" r=1,\n",
" lora_alpha=2,\n",
" lora_dropout=0.00,\n",
" task_type=\"CAUSAL_LM\",\n",
" target_modules=[\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\", \"fc1\", \"fc2\"]\n",
")\n",
"\n",
"model = get_peft_model(model, lora_config)\n",
"model.print_trainable_parameters()\n"
]
},
{
"cell_type": "markdown",
"id": "3f3533dd",
"metadata": {
"id": "3f3533dd"
},
"source": [
"## Demo the model\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0f930711",
"metadata": {
"id": "0f930711",
"outputId": "f263ab12-9b2c-4ea3-da1c-4465032538d2"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"================================================================================\n",
"TEST 1: Plain Autoregressive Prompt\n",
"================================================================================\n",
"Prompt: The Eiffel Tower stands in Paris and\n",
"\n",
"Generated: is one of the most famous landmarks in the world. It is located on the Champ de Mars in the heart of the city. The tower was built for the 1889 World's Fair. It was designed by the French engineer Gustave Eiffel and took 2 years to build. The Eiffel Tower stands 324 meters\n",
"================================================================================\n"
]
}
],
"source": [
"print(\"=\" * 80)\n",
"print(\"TEST 1: Plain Autoregressive Prompt\")\n",
"print(\"=\" * 80)\n",
"prompt = \"The Eiffel Tower stands in Paris and\"\n",
"test_inputs = tokenizer(prompt, return_tensors=\"pt\").to(device)\n",
"\n",
"\n",
"with torch.no_grad():\n",
" test_outputs = model.generate(\n",
" **test_inputs,\n",
" max_new_tokens=64,\n",
" do_sample=False,\n",
" pad_token_id=tokenizer.pad_token_id,\n",
" )\n",
"\n",
"generated_tokens = test_outputs[0, test_inputs[\"input_ids\"].shape[1] :]\n",
"print(f\"Prompt: {prompt}\")\n",
"print(f\"\\nGenerated: {tokenizer.decode(generated_tokens, skip_special_tokens=True)}\")\n",
"print(\"=\" * 80)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fbf80e5f",
"metadata": {
"id": "fbf80e5f",
"outputId": "86af20b4-3b9f-4dad-ba09-5dbb0de0f18c"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"================================================================================\n",
"TEST 2: Chat Template\n",
"================================================================================\n",
"Formatted prompt: <|bos|><|user_start|>What is the capital of France?<|user_end|><|assistant_start|>\n",
"Input IDs: [65527, 65528, 1442, 309, 261, 3429, 281, 4215, 63, 65529, 65530]\n",
"\n",
"Generated: The capital of France is Paris.<|assistant_end|>\n",
"================================================================================\n"
]
}
],
"source": [
"print(\"=\" * 80)\n",
"print(\"TEST 2: Chat Template\")\n",
"print(\"=\"*80)\n",
"conversation = [\n",
" {\"role\": \"user\", \"content\": \"What is the capital of France?\"},\n",
"]\n",
"\n",
"inputs = tokenizer.apply_chat_template(\n",
" conversation, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors=\"pt\"\n",
").to(device)\n",
"\n",
"print(f\"Formatted prompt: {tokenizer.decode(inputs['input_ids'][0])}\")\n",
"print(f\"Input IDs: {inputs['input_ids'][0].tolist()}\")\n",
"\n",
"with torch.no_grad():\n",
" outputs = model.generate(\n",
" **inputs,\n",
" max_new_tokens=64,\n",
" do_sample=False\n",
" )\n",
"\n",
"generated_tokens = outputs[0, inputs[\"input_ids\"].shape[1] :]\n",
"print(f\"\\nGenerated: {tokenizer.decode(generated_tokens)}\")\n",
"print(\"=\" * 80)\n"
]
},
{
"cell_type": "markdown",
"id": "a102e248",
"metadata": {
"id": "a102e248"
},
"source": [
"## Dataset\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b07e3b95",
"metadata": {
"id": "b07e3b95",
"outputId": "3c42b4d4-6e4f-4622-94cd-adbe53efa238"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Generating train split: 100%|██████████| 52736/52736 [00:00<00:00, 1058243.18 examples/s]\n"
]
}
],
"source": [
"raw_dataset = load_dataset(\"HuggingFaceH4/OpenR1-Math-220k-default-verified\", split=\"train\")\n",
"splits = raw_dataset.train_test_split(test_size=0.1, seed=42)\n",
"train_dataset = splits[\"train\"]\n",
"eval_dataset = splits[\"test\"]\n"
]
},
{
"cell_type": "markdown",
"id": "21ec9078",
"metadata": {
"id": "21ec9078"
},
"source": [
"## Training Configuration\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "17a49557",
"metadata": {
"id": "17a49557"
},
"outputs": [],
"source": [
"max_train_steps = 50\n",
"prompt_batch_size = 1\n",
"num_generations = 4\n",
"max_new_tokens = 128\n",
"temperature = 1.0\n",
"top_k = 50\n",
"learning_rate = 5e-6\n",
"weight_decay = 0.0\n",
"epsilon = 0.2\n",
"gradient_accumulation_steps = 1\n",
"warmup_ratio = 0.1\n",
"logging_frequency = 5\n",
"max_train_samples = 1000\n",
"max_eval_samples = 100\n"
]
},
{
"cell_type": "markdown",
"id": "a8a12581",
"metadata": {
"id": "a8a12581"
},
"source": [
"## Reward Functions\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3f07953f",
"metadata": {
"id": "3f07953f"
},
"outputs": [],
"source": [
"import re\n",
"import numpy as np\n",
"import torch.nn.functional as F\n",
"from contextlib import nullcontext\n",
"\n",
"\n",
"def think_format_reward(completions):\n",
" \"\"\"\n",
" Reward function that checks if the reasoning process is enclosed within and tags.\n",
" Returns 1.0 if the format is correct, otherwise 0.0.\n",
" \"\"\"\n",
" pattern = r\"^(?!.*)(.*?).*$\"\n",
" matches = [re.match(pattern, content, re.DOTALL | re.MULTILINE) for content in completions]\n",
" return [1.0 if match else 0.0 for match in matches]\n",
"\n",
"\n",
"def accuracy_reward(completions, solutions):\n",
" \"\"\"\n",
" Reward function that checks if the completion matches the solution.\n",
" For simplicity, we'll do basic string matching here.\n",
" \"\"\"\n",
" rewards = []\n",
" for completion, solution in zip(completions, solutions):\n",
" # Simple string matching (normalized)\n",
" reward = 1.0 if solution.strip().lower() in completion.strip().lower() else 0.0\n",
" rewards.append(reward)\n",
" return rewards\n",
"\n",
"\n",
"def min_length_reward(completions, min_length=10):\n",
" \"\"\"\n",
" Reward function that checks if the completion is at least a certain length.\n",
" Returns 1.0 if the length is greater than or equal to the minimum length, otherwise 0.0.\n",
" \"\"\"\n",
" return [1.0 if len(completion) >= min_length else 0.0 for completion in completions]\n",
"\n",
"def combined_reward(completions, solutions):\n",
" \"\"\"\n",
" Combines format and accuracy rewards with equal weight.\n",
" \"\"\"\n",
" format_rewards = think_format_reward(completions)\n",
" accuracy_rewards = accuracy_reward(completions, solutions)\n",
" min_length_rewards = min_length_reward(completions)\n",
" return [np.mean([f, a, m]) for f, a, m in zip(format_rewards, accuracy_rewards, min_length_rewards)]"
]
},
{
"cell_type": "markdown",
"id": "b2299e86",
"metadata": {
"id": "b2299e86"
},
"source": [
"## Helper Functions\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b0f0e9e4",
"metadata": {
"id": "b0f0e9e4"
},
"outputs": [],
"source": [
"def per_token_log_probs(logits, labels):\n",
" logits = logits.float()\n",
" log_probs = F.log_softmax(logits, dim=-1)\n",
" return log_probs.gather(dim=-1, index=labels.unsqueeze(-1)).squeeze(-1)\n",
"\n",
"\n",
"def prepare_prompt(example, problem_key=\"problem\", solution_key=\"solution\"):\n",
" # Extract the messages (should be a list of dicts with 'role' and 'content')\n",
" prompt = example.get(problem_key, \"\")\n",
" messages = [{\"role\": \"user\", \"content\": prompt}]\n",
"\n",
" formatted = tokenizer.apply_chat_template(\n",
" messages,\n",
" add_generation_prompt=True,\n",
" truncation=True,\n",
" max_length=2048,\n",
" padding=False,\n",
" return_dict=True,\n",
" return_tensors=\"pt\",\n",
" )\n",
" return formatted[\"input_ids\"], formatted[\"attention_mask\"]\n",
"\n",
"\n",
"if device.type == \"cuda\":\n",
" autocast_ctx = torch.amp.autocast(device_type=\"cuda\", dtype=torch.bfloat16)\n",
"else:\n",
" autocast_ctx = nullcontext()\n"
]
},
{
"cell_type": "markdown",
"id": "2756b691",
"metadata": {
"id": "2756b691"
},
"source": [
"## Optimizer and Scheduler\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e0e05495",
"metadata": {
"id": "e0e05495"
},
"outputs": [],
"source": [
"optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n",
"total_update_steps = max_train_steps // gradient_accumulation_steps\n",
"warmup_steps = max(1, int(total_update_steps * warmup_ratio))\n",
"scheduler = get_linear_schedule_with_warmup(optimizer, warmup_steps, total_update_steps)\n"
]
},
{
"cell_type": "markdown",
"id": "5e2c7a2c",
"metadata": {
"id": "5e2c7a2c"
},
"source": [
"# The Training Loop\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "260f574c",
"metadata": {
"id": "260f574c",
"outputId": "b762165f-ed4a-4b22-cbb7-2fa203696ac3"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"step=0005 | loss=0.0000 | avg_reward=0.4000 | lr=0.00e+00\n",
"Sample eval completion: 3^4 - 11 and 3^6 - 17\n",
"step=0010 | loss=0.0000 | avg_reward=0.3333 | lr=0.00e+00\n",
"Sample eval completion: 11. \n",
"\n",
"This statement refers to an optimization problem where we seek to find the smallest prime \\( p\n",
"step=0015 | loss=0.0000 | avg_reward=0.4667 | lr=0.00e+00\n",
"Sample eval completion: What number has two prime factors, 1 and itself, without additional restrictions? One possible combi\n",
"step=0020 | loss=-0.0983 | avg_reward=0.4500 | lr=0.00e+00\n",
"Sample eval completion: \\[\\begin{bmatrix} 2 & 3\\\\ 6 & 11\\end{bmatrix} \\]\\[3^{a}-2^{b}\\left(\\frac{1^{a}}{a}\\right) \\left(\\fra\n",
"step=0025 | loss=-0.0979 | avg_reward=0.3333 | lr=0.00e+00\n",
"Sample eval completion: Let's examine the smallest prime \\( p \\) for which there do not exist non-negative integers \\( a, b \n",
"step=0030 | loss=-0.0000 | avg_reward=0.3667 | lr=0.00e+00\n",
"Sample eval completion: \n",
"Since \\( p = 23^2 + 7 \\) or \\( p \\ge 23^3 + 63 \\), and \\( p > 23 \\), we find that \\( p \\ge 9223 \\).\n",
"step=0035 | loss=0.0431 | avg_reward=0.4167 | lr=0.00e+00\n",
"Sample eval completion: \\[11 \\] = \\((3^5)\\), for all \\( a, b \\).\n",
"[asy]\n",
"import random;\n",
"import numpy as np;\n",
"\n",
"unitsize(1cm);\n",
"\n",
"d\n",
"step=0040 | loss=-0.0702 | avg_reward=0.5000 | lr=0.00e+00\n",
"Sample eval completion: 3^4 - 7\n",
"step=0045 | loss=0.0000 | avg_reward=0.3333 | lr=0.00e+00\n",
"Sample eval completion: 7.\n",
"step=0050 | loss=0.0000 | avg_reward=0.4000 | lr=0.00e+00\n",
"Sample eval completion: Here is the answer:\n",
"\n",
"The smallest prime \\( p \\) (where \\( p > 3 \\)) for which there do not exist non\n",
"Training complete.\n"
]
}
],
"source": [
"\n",
"# Sample dataset if needed\n",
"if max_train_samples is not None and len(train_dataset) > max_train_samples:\n",
" train_dataset = train_dataset.select(range(max_train_samples))\n",
"if max_eval_samples is not None and len(eval_dataset) > max_eval_samples:\n",
" eval_dataset = eval_dataset.select(range(max_eval_samples))\n",
"\n",
"model.train()\n",
"train_index = 0\n",
"global_step = 0\n",
"running_reward = 0.0\n",
"running_loss = 0.0\n",
"\n",
"for step in range(1, max_train_steps + 1):\n",
" example = train_dataset[train_index % len(train_dataset)]\n",
" train_index += 1\n",
"\n",
" prompt_ids, prompt_mask = prepare_prompt(example)\n",
" prompt_ids = prompt_ids.to(device)\n",
" prompt_mask = prompt_mask.to(device)\n",
" prompt_length = prompt_ids.shape[1]\n",
"\n",
" prompt_repeat = prompt_ids.repeat(num_generations, 1)\n",
" mask_repeat = prompt_mask.repeat(num_generations, 1)\n",
"\n",
" # Generate completions\n",
" model.eval()\n",
" with torch.no_grad():\n",
" generated = model.generate(\n",
" input_ids=prompt_repeat,\n",
" attention_mask=mask_repeat,\n",
" max_new_tokens=max_new_tokens,\n",
" do_sample=True,\n",
" temperature=temperature,\n",
" top_k=top_k,\n",
" pad_token_id=tokenizer.pad_token_id,\n",
" )\n",
" model.train()\n",
"\n",
" sequences = generated\n",
" attention_mask = (sequences != tokenizer.pad_token_id).long()\n",
" completion_mask = attention_mask.clone()\n",
" completion_mask[:, :prompt_length] = 0\n",
"\n",
" completion_tokens = sequences[:, prompt_length:]\n",
" completion_texts = tokenizer.batch_decode(completion_tokens, skip_special_tokens=True)\n",
"\n",
" # Get solution\n",
" solution = example.get(\"solution\", example.get(\"answer\", \"\"))\n",
" solutions = [solution] * num_generations\n",
"\n",
" # Compute rewards\n",
" rewards = combined_reward(completion_texts, solutions)\n",
" rewards = torch.tensor(rewards, dtype=torch.float32, device=device)\n",
" running_reward += rewards.mean().item()\n",
"\n",
" rewards_view = rewards.view(prompt_batch_size, num_generations)\n",
" mean_rewards = rewards_view.mean(dim=1, keepdim=True)\n",
" std_rewards = rewards_view.std(dim=1, keepdim=True)\n",
" std_rewards = torch.where(std_rewards > 0, std_rewards, torch.ones_like(std_rewards))\n",
" advantages = ((rewards_view - mean_rewards) / std_rewards).view(-1)\n",
"\n",
" labels = sequences[:, 1:].clone()\n",
" labels[attention_mask[:, 1:] == 0] = tokenizer.pad_token_id\n",
"\n",
" # Compute old log probs\n",
" with torch.no_grad():\n",
" with (autocast_ctx if device.type == \"cuda\" else nullcontext()):\n",
" old_outputs = model(\n",
" input_ids=sequences,\n",
" attention_mask=attention_mask,\n",
" use_cache=False,\n",
" )\n",
" old_log_probs = per_token_log_probs(old_outputs.logits[:, :-1], labels)\n",
"\n",
" valid_mask = (completion_mask[:, 1:] == 1) & (labels != tokenizer.pad_token_id)\n",
"\n",
" # Compute loss\n",
" optimizer.zero_grad(set_to_none=True)\n",
" with (autocast_ctx if device.type == \"cuda\" else nullcontext()):\n",
" outputs = model(\n",
" input_ids=sequences,\n",
" attention_mask=attention_mask,\n",
" use_cache=False,\n",
" )\n",
" log_probs = per_token_log_probs(outputs.logits[:, :-1], labels)\n",
"\n",
" ratio = (log_probs - old_log_probs).exp()\n",
" ratio = torch.where(valid_mask, ratio, torch.ones_like(ratio))\n",
" clipped_ratio = ratio.clamp(1.0 - epsilon, 1.0 + epsilon)\n",
"\n",
" adv = advantages.unsqueeze(1)\n",
" loss_unclipped = ratio * adv\n",
" loss_clipped = clipped_ratio * adv\n",
" per_token_loss = -torch.min(loss_unclipped, loss_clipped)\n",
" per_token_loss = torch.where(valid_mask, per_token_loss, torch.zeros_like(per_token_loss))\n",
"\n",
" denom = valid_mask.sum().clamp(min=1)\n",
" loss = per_token_loss.sum() / denom\n",
"\n",
" loss.backward()\n",
" torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n",
" optimizer.step()\n",
" scheduler.step()\n",
"\n",
" global_step += 1\n",
" running_loss += loss.item()\n",
"\n",
" if step % logging_frequency == 0:\n",
" avg_reward = running_reward / logging_frequency\n",
" avg_loss = running_loss / logging_frequency\n",
" current_lr = scheduler.get_last_lr()[0]\n",
" print(\n",
" f\"step={step:04d} | loss={avg_loss:.4f} | avg_reward={avg_reward:.4f} | lr={current_lr:.2e}\"\n",
" )\n",
" running_reward = 0.0\n",
" running_loss = 0.0\n",
"\n",
" # Sample evaluation\n",
" model.eval()\n",
" eval_example = eval_dataset[0]\n",
" prompt_ids, prompt_mask = prepare_prompt(eval_example)\n",
" with torch.no_grad():\n",
" eval_sequences = model.generate(\n",
" input_ids=prompt_ids.to(device),\n",
" attention_mask=prompt_mask.to(device),\n",
" max_new_tokens=max_new_tokens,\n",
" do_sample=True,\n",
" top_k=top_k,\n",
" temperature=temperature,\n",
" pad_token_id=tokenizer.pad_token_id,\n",
" )\n",
" model.train()\n",
" completion = eval_sequences[0, prompt_ids.shape[1] :]\n",
" print(\"Sample eval completion:\", tokenizer.decode(completion, skip_special_tokens=True)[:100])\n",
"\n",
"print(\"Training complete.\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2104662d",
"metadata": {
"id": "2104662d"
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.18"
},
"colab": {
"provenance": []
}
},
"nbformat": 4,
"nbformat_minor": 5
}