Add task-specific WiggleGPT checkpoints and results
Browse filesAdd SST-2 sentiment and RACE-middle reading checkpoints, each fine-tuned for three epochs from the original pretrained model.
Results: 90.1% SST-2 validation accuracy and 59.5% RACE-middle test accuracy.
Include the updated model card, evaluation code and prediction records.
- .gitattributes +1 -0
- README.md +174 -128
- task_adaptations/comparison.json +0 -0
- task_adaptations/manifest.json +23 -0
- task_adaptations/race_middle/README.md +15 -0
- task_adaptations/race_middle/metadata.json +260 -0
- task_adaptations/race_middle/model.pt +3 -0
- task_adaptations/sst2/README.md +15 -0
- task_adaptations/sst2/metadata.json +186 -0
- task_adaptations/sst2/model.pt +3 -0
- task_finetuning/.gitignore +8 -0
- task_finetuning/README.md +82 -0
- task_finetuning/WiggleGPT_Demonstration.pdf +3 -0
- task_finetuning/data/.gitkeep +1 -0
- task_finetuning/encode_task_data.py +125 -0
- task_finetuning/evaluate_tasks.py +81 -0
- task_finetuning/native/model_bio.py +536 -0
- task_finetuning/prepare_data.py +116 -0
- task_finetuning/requirements.txt +8 -0
- task_finetuning/results/WiggleGPT_Generation_Record.json +1647 -0
- task_finetuning/results/comparison.json +0 -0
- task_finetuning/results/dataset_manifest.json +137 -0
- task_finetuning/results/encoding_summary.json +167 -0
- task_finetuning/results/manifest.json +137 -0
- task_finetuning/results/race_middle_metrics.jsonl +110 -0
- task_finetuning/results/race_middle_provenance.json +130 -0
- task_finetuning/results/race_middle_resume_provenance.json +19 -0
- task_finetuning/results/sst2_metrics.jsonl +273 -0
- task_finetuning/results/sst2_provenance.json +93 -0
- task_finetuning/results/train_tasks_original_source.txt +449 -0
- task_finetuning/results/train_tasks_v2_original_source.txt +539 -0
- task_finetuning/results/train_tasks_v3_original_source.txt +569 -0
- task_finetuning/task_runtime.py +86 -0
- task_finetuning/tokenizer/gpt2.tiktoken +0 -0
- task_finetuning/train_config.json +37 -0
- task_finetuning/train_tasks.py +451 -0
- task_finetuning/train_tasks_v3.py +571 -0
.gitattributes
CHANGED
|
@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
model_architecture.png filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
model_architecture.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
task_finetuning/WiggleGPT_Demonstration.pdf filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
|
@@ -1,128 +1,174 @@
|
|
| 1 |
-
---
|
| 2 |
-
license: gpl-3.0
|
| 3 |
-
tags:
|
| 4 |
-
- pytorch
|
| 5 |
-
- gpt2
|
| 6 |
-
- transformer
|
| 7 |
-
- oscillating-activation
|
| 8 |
-
- bio-inspired
|
| 9 |
-
- language-model
|
| 10 |
-
language:
|
| 11 |
-
- en
|
| 12 |
-
datasets:
|
| 13 |
-
- openwebtext
|
| 14 |
-
- HuggingFaceTB/smoltalk
|
| 15 |
-
|
| 16 |
-
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
```
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
|
| 41 |
-
|
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
|
| 52 |
-
|
| 53 |
-
- **
|
| 54 |
-
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|-
|
| 61 |
-
|
|
| 62 |
-
|
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
|
| 67 |
-
|
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
``
|
| 95 |
-
|
| 96 |
-
##
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: gpl-3.0
|
| 3 |
+
tags:
|
| 4 |
+
- pytorch
|
| 5 |
+
- gpt2
|
| 6 |
+
- transformer
|
| 7 |
+
- oscillating-activation
|
| 8 |
+
- bio-inspired
|
| 9 |
+
- language-model
|
| 10 |
+
language:
|
| 11 |
+
- en
|
| 12 |
+
datasets:
|
| 13 |
+
- openwebtext
|
| 14 |
+
- HuggingFaceTB/smoltalk
|
| 15 |
+
- stanfordnlp/sst2
|
| 16 |
+
- ehovy/race
|
| 17 |
+
pipeline_tag: text-generation
|
| 18 |
+
---
|
| 19 |
+
|
| 20 |
+
# WiggleGPT
|
| 21 |
+
|
| 22 |
+
A 124M parameter transformer that challenges a 56-year-old assumption in neural network design.
|
| 23 |
+
|
| 24 |
+

|
| 25 |
+
|
| 26 |
+
## What Makes It Different?
|
| 27 |
+
|
| 28 |
+
Since Minsky and Papert's *Perceptrons* (1969), neural networks have relied on **monotonic activation functions** (Sigmoid, ReLU, GELU) — requiring multiple hidden layers to solve non-linearly separable problems like XOR.
|
| 29 |
+
|
| 30 |
+
WiggleGPT replaces monotonic activations with **learnable oscillating functions**, enabling single neurons to create multiple decision boundaries:
|
| 31 |
+
|
| 32 |
+
```
|
| 33 |
+
f(x) = sin(ωx + φ) · tanh(x) + baseline
|
| 34 |
+
```
|
| 35 |
+
|
| 36 |
+
Where ω (frequency) and φ (phase) are **learnable per-neuron parameters**.
|
| 37 |
+
|
| 38 |
+
## Results
|
| 39 |
+
|
| 40 |
+
| Model | Parameters | Val Loss | Notes |
|
| 41 |
+
|-------|------------|----------|-------|
|
| 42 |
+
| **WiggleGPT** | 124M | **3.1621** | Oscillating activation |
|
| 43 |
+
| GPT-2 | 124M | ~3.12 | Standard GELU baseline |
|
| 44 |
+
|
| 45 |
+
**Within 1.3% of GPT-2 performance** — proving oscillating activations are a functional drop-in replacement at scale.
|
| 46 |
+
|
| 47 |
+
### Task-Specific Fine-Tuning — 26 October 2026
|
| 48 |
+
|
| 49 |
+
I revisited the original pretrained WiggleGPT and fine-tuned it separately for sentiment classification and multiple-choice reading comprehension, following the task-adaptation approach used in [GPT-1](https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf).
|
| 50 |
+
|
| 51 |
+
| Task | Evaluation split | Before task training | After task training | Improvement |
|
| 52 |
+
|------|------------------|----------------------|---------------------|-------------|
|
| 53 |
+
| **SST-2 sentiment** | Public validation, 872 examples | 437/872 — 50.1% | **786/872 — 90.1%** | **+40.0 percentage points** |
|
| 54 |
+
| **RACE-middle reading** | Test, 1,436 questions | 385/1,436 — 26.8% | **854/1,436 — 59.5%** | **+32.7 percentage points** |
|
| 55 |
+
|
| 56 |
+
“Before” is the original pretrained backbone with a newly initialized task classifier. “After” is the validation-selected task-trained checkpoint using the same input format and classifier. These rows measure two independent adaptations from the original 600,000-step pretrained model. They are separate from the SmolTalk instruction model and the later Gosport chat adaptation.
|
| 57 |
+
|
| 58 |
+
Both tasks completed **three full epochs**. The entire native backbone was trained with classifier cross-entropy plus an auxiliary language-model loss weighted by 0.5, effective batch size 32, peak learning rate 6.25 × 10⁻⁵, 0.2% linear warmup and linear decay. The oscillating activation, RMSNorm and RoPE architecture were retained. Classification uses the final task-marker state with a 768→2 head for SST-2 and a shared 768→1 head across the four RACE answers.
|
| 59 |
+
|
| 60 |
+
| Training detail | SST-2 | RACE-middle |
|
| 61 |
+
|-----------------|-------|-------------|
|
| 62 |
+
| Training examples | 67,349 | 25,421 |
|
| 63 |
+
| Validation examples | 872 | 1,436 |
|
| 64 |
+
| Completed / best update | 6,315 / 6,315 | 2,385 / 2,385 |
|
| 65 |
+
| Task-head parameters | 1,538 | 769 |
|
| 66 |
+
| Total trained parameters | 123,699,458 | 123,698,689 |
|
| 67 |
+
| Stopping cause | Three epochs completed | Three epochs completed |
|
| 68 |
+
|
| 69 |
+
Checkpoint selection used highest validation accuracy, with classification loss breaking ties. SST-2 is reported on public validation because public test labels are hidden; that validation split also selected its checkpoint. RACE-middle test was evaluated after validation selection. Inputs were capped at 512 tokens. No SST-2 inputs were truncated; the article prefix was shortened for 210 training, 9 validation and 14 test RACE questions, preserving the complete question and answer choices.
|
| 70 |
+
|
| 71 |
+
For historical context, GPT-1 reported **91.3% on SST-2 GLUE test** and **62.9% on RACE-M**, with RACE training covering both middle- and high-school data. WiggleGPT's SST-2 row uses public validation and its RACE adaptation uses middle-school data only, so these historical figures describe different evaluation splits and training coverage.
|
| 72 |
+
|
| 73 |
+
Data: [stanfordnlp/sst2](https://huggingface.co/datasets/stanfordnlp/sst2), [ehovy/race — middle](https://huggingface.co/datasets/ehovy/race). The [shareable comparison record](task_adaptations/comparison.json) retains every evaluated ID, gold label, before/after prediction, correctness flag and source-text hash. [Checkpoint metadata](task_adaptations/manifest.json) records the inference exports and their SHA-256 hashes.
|
| 74 |
+
|
| 75 |
+
### The Model Actually Learned to Oscillate
|
| 76 |
+
|
| 77 |
+
| Parameter | Init | After Training | Change |
|
| 78 |
+
|-----------|------|----------------|--------|
|
| 79 |
+
| ω mean | 1.0 | 1.096 | +9.6% |
|
| 80 |
+
| ω std | 0.1 | **0.602** | **6× increase** |
|
| 81 |
+
| ω range | [0.8, 1.2] | [-0.19, 5.17] | Massive expansion |
|
| 82 |
+
|
| 83 |
+
- **95% of neurons retained active oscillation** (ω > 0.1)
|
| 84 |
+
- Some neurons learned frequencies up to ω = 5.17 (five oscillations per unit input)
|
| 85 |
+
- Full phase coverage [-π, +π] after training
|
| 86 |
+
|
| 87 |
+
## Checkpoints
|
| 88 |
+
|
| 89 |
+
| File | Description |
|
| 90 |
+
|------|-------------|
|
| 91 |
+
| `ckpt_pretrain.pt` | Base model trained on OpenWebText (~600k iterations) |
|
| 92 |
+
| `ckpt_finetune.pt` | Fine-tuned on SmolTalk2 (instruction following) |
|
| 93 |
+
| `task_adaptations/sst2/model.pt` | Independent full-model SST-2 adaptation with sentiment classifier; inference-only export |
|
| 94 |
+
| `task_adaptations/race_middle/model.pt` | Independent full-model RACE-middle adaptation with four-choice classifier; inference-only export |
|
| 95 |
+
|
| 96 |
+
## Architecture
|
| 97 |
+
|
| 98 |
+
| Component | Specification |
|
| 99 |
+
|-----------|---------------|
|
| 100 |
+
| Parameters | 123,697,920 |
|
| 101 |
+
| Layers | 12 |
|
| 102 |
+
| Attention Heads | 12 |
|
| 103 |
+
| Embedding Dimension | 768 |
|
| 104 |
+
| Oscillating Neurons | 36,864 (each with learnable ω, φ, baseline) |
|
| 105 |
+
| Normalization | RMSNorm |
|
| 106 |
+
| Position Encoding | RoPE (Rotary) |
|
| 107 |
+
| Attention | Flash Attention (when available) |
|
| 108 |
+
|
| 109 |
+
## Usage
|
| 110 |
+
|
| 111 |
+
See the [GitHub repository](https://github.com/Eden-Eldith/WiggleGPT) for full training, inference, and chat scripts.
|
| 112 |
+
|
| 113 |
+
```python
|
| 114 |
+
# Quick inference example
|
| 115 |
+
import torch
|
| 116 |
+
from model_bio import GPT, GPTConfig
|
| 117 |
+
|
| 118 |
+
# Load checkpoint
|
| 119 |
+
checkpoint = torch.load('ckpt_pretrain.pt', map_location='cpu', weights_only=False)
|
| 120 |
+
config = GPTConfig(**checkpoint['model_args'])
|
| 121 |
+
with torch.device('cuda:0'):
|
| 122 |
+
model = GPT(config)
|
| 123 |
+
state = {key.removeprefix('_orig_mod.'): value for key, value in checkpoint['model'].items()}
|
| 124 |
+
model.load_state_dict(state, strict=True)
|
| 125 |
+
model.eval()
|
| 126 |
+
|
| 127 |
+
# Generate text (see sample_bio.py for full implementation)
|
| 128 |
+
```
|
| 129 |
+
|
| 130 |
+
### Task Classifier Usage
|
| 131 |
+
|
| 132 |
+
The task checkpoints contain the full adapted backbone and a custom classifier. They use the native WiggleGPT wrapper; they are not Transformers `AutoModel` checkpoints. They are inference exports, with optimizer and RNG state removed.
|
| 133 |
+
|
| 134 |
+
Download `task_adaptations/` and the accompanying `task_finetuning/` runtime, then use the bundled command-line interface:
|
| 135 |
+
|
| 136 |
+
```bash
|
| 137 |
+
python task_finetuning/evaluate_tasks.py --help
|
| 138 |
+
```
|
| 139 |
+
|
| 140 |
+
The [task runtime guide](task_finetuning/README.md) gives exact commands for task scoring and re-evaluation. Each export also includes `model_args`, the marker IDs, original tokenizer identity, task input layout, training configuration and metric metadata. Use the accompanying exact native model snapshot in `task_finetuning/native/model_bio.py`.
|
| 141 |
+
|
| 142 |
+
## Training Details
|
| 143 |
+
|
| 144 |
+
**Pretraining:**
|
| 145 |
+
- Dataset: OpenWebText (~9B tokens)
|
| 146 |
+
- Iterations: 600,000
|
| 147 |
+
- Hardware: RTX 3070 (steps 0–354k) → RTX 5060 Ti 16GB (steps 354k–600k)
|
| 148 |
+
- Time: Roughly 20 days total (~15 days on 3070, ~5 days on 5060 Ti)
|
| 149 |
+
|
| 150 |
+
**Fine-tuning:**
|
| 151 |
+
- Dataset: SmolTalk2 (406K examples)
|
| 152 |
+
- Oscillation parameters (ω, φ) remained stable — 0.0% of neurons shifted by >0.1
|
| 153 |
+
|
| 154 |
+
## Citation
|
| 155 |
+
|
| 156 |
+
```bibtex
|
| 157 |
+
@software{wigglegpt2025,
|
| 158 |
+
author = {O'Brien, Phillip C.},
|
| 159 |
+
title = {WiggleGPT: Revisiting the Monotonicity Assumption in Neural Networks via Oscillating Activation Functions},
|
| 160 |
+
year = {2025},
|
| 161 |
+
url = {https://github.com/Eden-Eldith/WiggleGPT}
|
| 162 |
+
}
|
| 163 |
+
```
|
| 164 |
+
|
| 165 |
+
## Author
|
| 166 |
+
|
| 167 |
+
**Eden (Phillip C. O'Brien)**
|
| 168 |
+
Independent AI Researcher | ORCID: [0009-0007-3961-1182](https://orcid.org/0009-0007-3961-1182)
|
| 169 |
+
|
| 170 |
+
Built in a garage lab in Gosport, UK. No academic affiliation, no institutional funding — just curiosity and an RTX 3070.
|
| 171 |
+
|
| 172 |
+
## License
|
| 173 |
+
|
| 174 |
+
GPL-3.0 — if you build on this, keep it open source.
|
task_adaptations/comparison.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
task_adaptations/manifest.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "wigglegpt_task_exports_v1",
|
| 3 |
+
"date": "2026-10-26",
|
| 4 |
+
"method": "Each task starts from the original pretrained WiggleGPT. All model weights are adapted using classifier cross-entropy plus an auxiliary language-model loss (weight 0.5), with GPT-1-style task markers and linear classifiers.",
|
| 5 |
+
"control_method": "Before: original pretrained backbone with a newly initialized task classifier. After: the validation-selected task-trained checkpoint, using the same inputs and classifier format.",
|
| 6 |
+
"original_backbone_parameters": 123697920,
|
| 7 |
+
"files": [
|
| 8 |
+
{
|
| 9 |
+
"path": "task_adaptations/sst2/model.pt",
|
| 10 |
+
"bytes": 494839923,
|
| 11 |
+
"sha256": "0a0c73b9ec95da128c07abe6949abd2db8492c829f52bd7bb350f67aef684c66",
|
| 12 |
+
"source_training_checkpoint_sha256": "2e2630c24231098e9381b09b2e6f3d27a273696743bfe65bffe4d5f2ecd4d80e",
|
| 13 |
+
"all_tensors_equal_to_source": true
|
| 14 |
+
},
|
| 15 |
+
{
|
| 16 |
+
"path": "task_adaptations/race_middle/model.pt",
|
| 17 |
+
"bytes": 494839091,
|
| 18 |
+
"sha256": "214d8252e78dccb8e3f5c0549cc99507176b09a55ee4c0bd04b168c829071855",
|
| 19 |
+
"source_training_checkpoint_sha256": "ff98d64794bd9e9a4514a05ba3441d827e9fe1530b371776d0a6030235bdbfed",
|
| 20 |
+
"all_tensors_equal_to_source": true
|
| 21 |
+
}
|
| 22 |
+
]
|
| 23 |
+
}
|
task_adaptations/race_middle/README.md
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# WiggleGPT — RACE-middle reading comprehension
|
| 2 |
+
|
| 3 |
+
Independent full-model adaptation from the original 600,000-step pretrained WiggleGPT.
|
| 4 |
+
|
| 5 |
+
- Scored split: test.
|
| 6 |
+
- Before task training: 385/1436 (26.8%).
|
| 7 |
+
- After three epochs: **854/1436 (59.5%)**.
|
| 8 |
+
- Best update: 2,385.
|
| 9 |
+
- Format: `wigglegpt_task_inference_v1`; full native backbone plus custom task head.
|
| 10 |
+
- File: `model.pt`, 494,839,091 bytes.
|
| 11 |
+
- SHA-256: `214d8252e78dccb8e3f5c0549cc99507176b09a55ee4c0bd04b168c829071855`.
|
| 12 |
+
|
| 13 |
+
The before score uses a newly initialized task classifier. The after checkpoint was selected by validation accuracy. Original language-model checkpoints are separate.
|
| 14 |
+
|
| 15 |
+
Use the accompanying [task runtime guide](../../task_finetuning/README.md) and [metadata.json](metadata.json). The export contains `model`, `model_args`, `task`, `markers`, `config`, `task_config`, `training`, `metrics` and `provenance`. It omits optimizer and RNG state. Tensor values were checked exactly against the completed best checkpoint.
|
task_adaptations/race_middle/metadata.json
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "wigglegpt_task_inference_v1",
|
| 3 |
+
"task": "race_middle",
|
| 4 |
+
"model_args": {
|
| 5 |
+
"block_size": 1024,
|
| 6 |
+
"vocab_size": 50304,
|
| 7 |
+
"n_layer": 12,
|
| 8 |
+
"n_head": 12,
|
| 9 |
+
"n_embd": 768,
|
| 10 |
+
"dropout": 0.0,
|
| 11 |
+
"bias": false,
|
| 12 |
+
"use_bio_mlp": true,
|
| 13 |
+
"use_rmsnorm": true,
|
| 14 |
+
"use_rope": true,
|
| 15 |
+
"gradient_checkpointing": false
|
| 16 |
+
},
|
| 17 |
+
"markers": {
|
| 18 |
+
"start": 50257,
|
| 19 |
+
"delimiter": 50258,
|
| 20 |
+
"classify": 50259,
|
| 21 |
+
"pad": 50256
|
| 22 |
+
},
|
| 23 |
+
"config": {
|
| 24 |
+
"starting_checkpoint": "ckpt_pretrain.pt (original 600,000-step pretrained model)",
|
| 25 |
+
"epochs": 3,
|
| 26 |
+
"effective_batch_size": 32,
|
| 27 |
+
"learning_rate": 6.25e-05,
|
| 28 |
+
"warmup_fraction": 0.002,
|
| 29 |
+
"schedule": "linear warmup then linear decay, matching GPT-1 task adaptation",
|
| 30 |
+
"lm_coefficient": 0.5,
|
| 31 |
+
"classifier_dropout": 0.1,
|
| 32 |
+
"weight_decay": 0.01,
|
| 33 |
+
"betas": [
|
| 34 |
+
0.9,
|
| 35 |
+
0.999
|
| 36 |
+
],
|
| 37 |
+
"epsilon": 1e-08,
|
| 38 |
+
"grad_clip": 1.0,
|
| 39 |
+
"max_length": 512,
|
| 40 |
+
"seed": 42,
|
| 41 |
+
"cpu_threads": 2,
|
| 42 |
+
"desktop_reserve_gib": 2.5,
|
| 43 |
+
"max_process_gib": 9.0,
|
| 44 |
+
"checkpoint_interval": 500,
|
| 45 |
+
"log_interval": 25,
|
| 46 |
+
"max_consecutive_nonfinite": 8,
|
| 47 |
+
"tasks": {
|
| 48 |
+
"sst2": {
|
| 49 |
+
"microbatch_size": 16,
|
| 50 |
+
"gradient_checkpointing": false,
|
| 51 |
+
"eval_interval": 1000
|
| 52 |
+
},
|
| 53 |
+
"race_middle": {
|
| 54 |
+
"microbatch_size": 2,
|
| 55 |
+
"gradient_checkpointing": true,
|
| 56 |
+
"eval_interval": 400
|
| 57 |
+
}
|
| 58 |
+
}
|
| 59 |
+
},
|
| 60 |
+
"task_config": {
|
| 61 |
+
"max_length": 512,
|
| 62 |
+
"classifier_dropout": 0.1,
|
| 63 |
+
"label_names": [
|
| 64 |
+
"A",
|
| 65 |
+
"B",
|
| 66 |
+
"C",
|
| 67 |
+
"D"
|
| 68 |
+
],
|
| 69 |
+
"head": "shared 768-to-1 linear over four answers",
|
| 70 |
+
"task_head_parameters": 769,
|
| 71 |
+
"input_layout": "[start] article_prefix [delimiter] question [delimiter] answer [classify]",
|
| 72 |
+
"truncation": "same article prefix for all four choices, budgeted using longest answer; question and choices preserved",
|
| 73 |
+
"tokenizer": "GPT-2 r50k_base",
|
| 74 |
+
"tokenizer_merges_sha256": "306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930",
|
| 75 |
+
"right_padding": true,
|
| 76 |
+
"minimum_padded_width": 16,
|
| 77 |
+
"classification_position": "last non-padding token (classify marker)",
|
| 78 |
+
"state_key_prefixes": [
|
| 79 |
+
"base.",
|
| 80 |
+
"classifier."
|
| 81 |
+
],
|
| 82 |
+
"requires_custom_wrapper": true
|
| 83 |
+
},
|
| 84 |
+
"training": {
|
| 85 |
+
"epochs": 3,
|
| 86 |
+
"max_steps": 2385,
|
| 87 |
+
"steps": 2385,
|
| 88 |
+
"batch_size": 32,
|
| 89 |
+
"microbatch_size": 2,
|
| 90 |
+
"gradient_accumulation": 16,
|
| 91 |
+
"learning_rate": 6.25e-05,
|
| 92 |
+
"max_length": 512,
|
| 93 |
+
"lm_coefficient": 0.5,
|
| 94 |
+
"stopping_cause": "epochs_complete",
|
| 95 |
+
"classifier_dropout": 0.1,
|
| 96 |
+
"native_backbone_dropout": 0.0,
|
| 97 |
+
"nonfinite_updates": 0,
|
| 98 |
+
"best_step": 2385,
|
| 99 |
+
"peak_allocated_gib": 6.383,
|
| 100 |
+
"trainable_parameters": 123698689,
|
| 101 |
+
"head": "shared 768-to-1 linear over four answers",
|
| 102 |
+
"checkpoint_selection": "highest validation accuracy, lower validation classification loss breaks ties",
|
| 103 |
+
"optimizations": {
|
| 104 |
+
"version": 3,
|
| 105 |
+
"training_padding": "CPU epoch plan crops each unchanged microbatch to its longest candidate, rounded up to 16 tokens (minimum 16).",
|
| 106 |
+
"auxiliary_lm": "Project only valid next-token positions; weight each token by the inverse length of its own candidate, then average all candidates.",
|
| 107 |
+
"device_synchronization": "One order copy to CPU per epoch; CPU-known output_size avoids synchronization when building packed CUDA indices.",
|
| 108 |
+
"question_order_and_effective_batch": "Original seeded CUDA randperm, effective batch and microbatch membership retained.",
|
| 109 |
+
"native_backbone": "Unchanged native model_bio.py; RACE checkpointing enabled when cropped width exceeds 384 and disabled at or below 384. SST-2 keeps its configured setting."
|
| 110 |
+
},
|
| 111 |
+
"adaptive_checkpointing_microbatches": {
|
| 112 |
+
"enabled": 5780,
|
| 113 |
+
"disabled": 25953
|
| 114 |
+
},
|
| 115 |
+
"resume_segments": [
|
| 116 |
+
"resume_segments\\2026-10-26_06-36-08_291546_train_tasks_v3"
|
| 117 |
+
],
|
| 118 |
+
"packed_loss_verification": {
|
| 119 |
+
"passed": true,
|
| 120 |
+
"dense_loss": 5.025767803192139,
|
| 121 |
+
"packed_loss": 5.0257673263549805,
|
| 122 |
+
"absolute_difference": 4.76837158203125e-07,
|
| 123 |
+
"relative_tolerance": 0.0001,
|
| 124 |
+
"absolute_tolerance": 1e-05,
|
| 125 |
+
"candidates_checked": 1,
|
| 126 |
+
"actual_training_candidates": 8,
|
| 127 |
+
"packed_lm_tokens": 2525,
|
| 128 |
+
"cropped_width": 432,
|
| 129 |
+
"projection_dtype": "torch.bfloat16",
|
| 130 |
+
"device": "cuda:0",
|
| 131 |
+
"comparison": "Same actual training hidden states; dense masked CE versus packed valid-position CE, before the first optimizer update."
|
| 132 |
+
},
|
| 133 |
+
"details": "Full native backbone trained. Zero decay on oscillator and 1-D parameters. Linear GPT-1 warmup/decay, AdamW, gradient clip1.0. Original backbone dropout retained; classifier dropout0.1. Three unused padding-vocabulary slots initialized for task markers. V2 padding cropping and packed LM projection retained. V3 RACE native gradient checkpointing is enabled above 384 cropped tokens and disabled at or below 384; counters cover v3 segments, while earlier v2 segments used checkpointing throughout. Question order, effective batch and candidate-wise loss normalization are preserved.",
|
| 134 |
+
"data_truncation": {
|
| 135 |
+
"train": {
|
| 136 |
+
"examples_with_truncation": 210,
|
| 137 |
+
"candidate_sequences_with_truncation": 840,
|
| 138 |
+
"total_tokens_removed": 10052,
|
| 139 |
+
"max_tokens_removed_per_example": 161
|
| 140 |
+
},
|
| 141 |
+
"validation": {
|
| 142 |
+
"examples_with_truncation": 9,
|
| 143 |
+
"candidate_sequences_with_truncation": 36,
|
| 144 |
+
"total_tokens_removed": 71,
|
| 145 |
+
"max_tokens_removed_per_example": 18
|
| 146 |
+
},
|
| 147 |
+
"test": {
|
| 148 |
+
"examples_with_truncation": 14,
|
| 149 |
+
"candidate_sequences_with_truncation": 56,
|
| 150 |
+
"total_tokens_removed": 531,
|
| 151 |
+
"max_tokens_removed_per_example": 53
|
| 152 |
+
}
|
| 153 |
+
},
|
| 154 |
+
"segments": 2
|
| 155 |
+
},
|
| 156 |
+
"metrics": {
|
| 157 |
+
"heldout_split": "test",
|
| 158 |
+
"before": {
|
| 159 |
+
"label": "Before task training (same classifier)",
|
| 160 |
+
"correct": 385,
|
| 161 |
+
"total": 1436,
|
| 162 |
+
"accuracy": 0.2681058495821727,
|
| 163 |
+
"loss": 2.6538400650024414,
|
| 164 |
+
"gold_label_counts": [
|
| 165 |
+
309,
|
| 166 |
+
388,
|
| 167 |
+
374,
|
| 168 |
+
365
|
| 169 |
+
],
|
| 170 |
+
"majority_accuracy": 0.27019498607242337
|
| 171 |
+
},
|
| 172 |
+
"after": {
|
| 173 |
+
"correct": 854,
|
| 174 |
+
"total": 1436,
|
| 175 |
+
"accuracy": 0.5947075208913649,
|
| 176 |
+
"loss": 2.4820992946624756,
|
| 177 |
+
"gold_label_counts": [
|
| 178 |
+
309,
|
| 179 |
+
388,
|
| 180 |
+
374,
|
| 181 |
+
365
|
| 182 |
+
],
|
| 183 |
+
"majority_accuracy": 0.27019498607242337
|
| 184 |
+
},
|
| 185 |
+
"confusion_matrix": {
|
| 186 |
+
"before": [
|
| 187 |
+
[
|
| 188 |
+
78,
|
| 189 |
+
65,
|
| 190 |
+
81,
|
| 191 |
+
85
|
| 192 |
+
],
|
| 193 |
+
[
|
| 194 |
+
98,
|
| 195 |
+
108,
|
| 196 |
+
100,
|
| 197 |
+
82
|
| 198 |
+
],
|
| 199 |
+
[
|
| 200 |
+
85,
|
| 201 |
+
97,
|
| 202 |
+
102,
|
| 203 |
+
90
|
| 204 |
+
],
|
| 205 |
+
[
|
| 206 |
+
89,
|
| 207 |
+
79,
|
| 208 |
+
100,
|
| 209 |
+
97
|
| 210 |
+
]
|
| 211 |
+
],
|
| 212 |
+
"after": [
|
| 213 |
+
[
|
| 214 |
+
170,
|
| 215 |
+
47,
|
| 216 |
+
42,
|
| 217 |
+
50
|
| 218 |
+
],
|
| 219 |
+
[
|
| 220 |
+
60,
|
| 221 |
+
220,
|
| 222 |
+
49,
|
| 223 |
+
59
|
| 224 |
+
],
|
| 225 |
+
[
|
| 226 |
+
50,
|
| 227 |
+
44,
|
| 228 |
+
238,
|
| 229 |
+
42
|
| 230 |
+
],
|
| 231 |
+
[
|
| 232 |
+
49,
|
| 233 |
+
53,
|
| 234 |
+
37,
|
| 235 |
+
226
|
| 236 |
+
]
|
| 237 |
+
]
|
| 238 |
+
},
|
| 239 |
+
"train_examples": 25421,
|
| 240 |
+
"validation_examples": 1436,
|
| 241 |
+
"test_examples": 1436
|
| 242 |
+
},
|
| 243 |
+
"provenance": {
|
| 244 |
+
"source_training_checkpoint": "best.pt",
|
| 245 |
+
"source_training_checkpoint_sha256": "ff98d64794bd9e9a4514a05ba3441d827e9fe1530b371776d0a6030235bdbfed",
|
| 246 |
+
"source_hashes": {
|
| 247 |
+
"model": "a28201b8194941a75d3ca2e0812e6d2d0eef1b2656cc0c9f8fd0ca65bf96b28b",
|
| 248 |
+
"trainer": "080b06025459b59c2dddb2be046f28c5906e40915dbbb325a8c64642511d5517"
|
| 249 |
+
},
|
| 250 |
+
"original_pretrained_checkpoint_sha256": "88afaab6c93ef3e33940b4e25cbce2f44038c428109600afe8285586a3a767b9",
|
| 251 |
+
"original_backbone_unchanged_during_export": true,
|
| 252 |
+
"task_weights": "independent full-model task adaptation of original pretrained backbone",
|
| 253 |
+
"export": "CPU state-dict serialization only; optimizer, order and RNG removed; no model construction or forward pass",
|
| 254 |
+
"torch_version": "2.11.0.dev20260215+cu128"
|
| 255 |
+
},
|
| 256 |
+
"file": "model.pt",
|
| 257 |
+
"bytes": 494839091,
|
| 258 |
+
"sha256": "214d8252e78dccb8e3f5c0549cc99507176b09a55ee4c0bd04b168c829071855",
|
| 259 |
+
"validation": "All model tensors exactly equal to the completed best checkpoint; source SHA-256 matches comparison record."
|
| 260 |
+
}
|
task_adaptations/race_middle/model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:214d8252e78dccb8e3f5c0549cc99507176b09a55ee4c0bd04b168c829071855
|
| 3 |
+
size 494839091
|
task_adaptations/sst2/README.md
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# WiggleGPT — SST-2 sentiment
|
| 2 |
+
|
| 3 |
+
Independent full-model adaptation from the original 600,000-step pretrained WiggleGPT.
|
| 4 |
+
|
| 5 |
+
- Scored split: validation.
|
| 6 |
+
- Before task training: 437/872 (50.1%).
|
| 7 |
+
- After three epochs: **786/872 (90.1%)**.
|
| 8 |
+
- Best update: 6,315.
|
| 9 |
+
- Format: `wigglegpt_task_inference_v1`; full native backbone plus custom task head.
|
| 10 |
+
- File: `model.pt`, 494,839,923 bytes.
|
| 11 |
+
- SHA-256: `0a0c73b9ec95da128c07abe6949abd2db8492c829f52bd7bb350f67aef684c66`.
|
| 12 |
+
|
| 13 |
+
The before score uses a newly initialized task classifier. The after checkpoint was selected by validation accuracy. Original language-model checkpoints are separate.
|
| 14 |
+
|
| 15 |
+
Use the accompanying [task runtime guide](../../task_finetuning/README.md) and [metadata.json](metadata.json). The export contains `model`, `model_args`, `task`, `markers`, `config`, `task_config`, `training`, `metrics` and `provenance`. It omits optimizer and RNG state. Tensor values were checked exactly against the completed best checkpoint.
|
task_adaptations/sst2/metadata.json
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "wigglegpt_task_inference_v1",
|
| 3 |
+
"task": "sst2",
|
| 4 |
+
"model_args": {
|
| 5 |
+
"block_size": 1024,
|
| 6 |
+
"vocab_size": 50304,
|
| 7 |
+
"n_layer": 12,
|
| 8 |
+
"n_head": 12,
|
| 9 |
+
"n_embd": 768,
|
| 10 |
+
"dropout": 0.0,
|
| 11 |
+
"bias": false,
|
| 12 |
+
"use_bio_mlp": true,
|
| 13 |
+
"use_rmsnorm": true,
|
| 14 |
+
"use_rope": true,
|
| 15 |
+
"gradient_checkpointing": false
|
| 16 |
+
},
|
| 17 |
+
"markers": {
|
| 18 |
+
"start": 50257,
|
| 19 |
+
"delimiter": 50258,
|
| 20 |
+
"classify": 50259,
|
| 21 |
+
"pad": 50256
|
| 22 |
+
},
|
| 23 |
+
"config": {
|
| 24 |
+
"starting_checkpoint": "ckpt_pretrain.pt (original 600,000-step pretrained model)",
|
| 25 |
+
"epochs": 3,
|
| 26 |
+
"effective_batch_size": 32,
|
| 27 |
+
"learning_rate": 6.25e-05,
|
| 28 |
+
"warmup_fraction": 0.002,
|
| 29 |
+
"schedule": "linear warmup then linear decay, matching GPT-1 task adaptation",
|
| 30 |
+
"lm_coefficient": 0.5,
|
| 31 |
+
"classifier_dropout": 0.1,
|
| 32 |
+
"weight_decay": 0.01,
|
| 33 |
+
"betas": [
|
| 34 |
+
0.9,
|
| 35 |
+
0.999
|
| 36 |
+
],
|
| 37 |
+
"epsilon": 1e-08,
|
| 38 |
+
"grad_clip": 1.0,
|
| 39 |
+
"max_length": 512,
|
| 40 |
+
"seed": 42,
|
| 41 |
+
"cpu_threads": 2,
|
| 42 |
+
"desktop_reserve_gib": 2.5,
|
| 43 |
+
"max_process_gib": 9.0,
|
| 44 |
+
"checkpoint_interval": 500,
|
| 45 |
+
"log_interval": 25,
|
| 46 |
+
"max_consecutive_nonfinite": 8,
|
| 47 |
+
"tasks": {
|
| 48 |
+
"sst2": {
|
| 49 |
+
"microbatch_size": 16,
|
| 50 |
+
"gradient_checkpointing": false,
|
| 51 |
+
"eval_interval": 1000
|
| 52 |
+
},
|
| 53 |
+
"race_middle": {
|
| 54 |
+
"microbatch_size": 2,
|
| 55 |
+
"gradient_checkpointing": true,
|
| 56 |
+
"eval_interval": 400
|
| 57 |
+
}
|
| 58 |
+
}
|
| 59 |
+
},
|
| 60 |
+
"task_config": {
|
| 61 |
+
"max_length": 512,
|
| 62 |
+
"classifier_dropout": 0.1,
|
| 63 |
+
"label_names": [
|
| 64 |
+
"negative",
|
| 65 |
+
"positive"
|
| 66 |
+
],
|
| 67 |
+
"head": "768-to-2 linear",
|
| 68 |
+
"task_head_parameters": 1538,
|
| 69 |
+
"input_layout": "[start] review [classify]",
|
| 70 |
+
"truncation": "review tokens truncated only if required to fit 512 total tokens",
|
| 71 |
+
"tokenizer": "GPT-2 r50k_base",
|
| 72 |
+
"tokenizer_merges_sha256": "306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930",
|
| 73 |
+
"right_padding": true,
|
| 74 |
+
"minimum_padded_width": 16,
|
| 75 |
+
"classification_position": "last non-padding token (classify marker)",
|
| 76 |
+
"state_key_prefixes": [
|
| 77 |
+
"base.",
|
| 78 |
+
"classifier."
|
| 79 |
+
],
|
| 80 |
+
"requires_custom_wrapper": true
|
| 81 |
+
},
|
| 82 |
+
"training": {
|
| 83 |
+
"epochs": 3,
|
| 84 |
+
"max_steps": 6315,
|
| 85 |
+
"steps": 6315,
|
| 86 |
+
"batch_size": 32,
|
| 87 |
+
"microbatch_size": 16,
|
| 88 |
+
"gradient_accumulation": 2,
|
| 89 |
+
"learning_rate": 6.25e-05,
|
| 90 |
+
"max_length": 512,
|
| 91 |
+
"lm_coefficient": 0.5,
|
| 92 |
+
"stopping_cause": "epochs_complete",
|
| 93 |
+
"classifier_dropout": 0.1,
|
| 94 |
+
"native_backbone_dropout": 0.0,
|
| 95 |
+
"nonfinite_updates": 0,
|
| 96 |
+
"best_step": 6315,
|
| 97 |
+
"peak_allocated_gib": 3.846,
|
| 98 |
+
"trainable_parameters": 123699458,
|
| 99 |
+
"head": "768-to-2 linear",
|
| 100 |
+
"checkpoint_selection": "highest validation accuracy, lower validation classification loss breaks ties",
|
| 101 |
+
"details": "Full native backbone trained. Zero decay on oscillator and 1-D parameters. Linear GPT-1 warmup/decay, AdamW, gradient clip1.0. Original backbone dropout retained; classifier dropout0.1. Three unused padding-vocabulary slots initialized for task markers.",
|
| 102 |
+
"data_truncation": {
|
| 103 |
+
"train": {
|
| 104 |
+
"examples_with_truncation": 0,
|
| 105 |
+
"candidate_sequences_with_truncation": 0,
|
| 106 |
+
"total_tokens_removed": 0,
|
| 107 |
+
"max_tokens_removed_per_example": 0
|
| 108 |
+
},
|
| 109 |
+
"validation": {
|
| 110 |
+
"examples_with_truncation": 0,
|
| 111 |
+
"candidate_sequences_with_truncation": 0,
|
| 112 |
+
"total_tokens_removed": 0,
|
| 113 |
+
"max_tokens_removed_per_example": 0
|
| 114 |
+
}
|
| 115 |
+
},
|
| 116 |
+
"segments": 2
|
| 117 |
+
},
|
| 118 |
+
"metrics": {
|
| 119 |
+
"heldout_split": "validation",
|
| 120 |
+
"before": {
|
| 121 |
+
"label": "Before task training (same classifier)",
|
| 122 |
+
"correct": 437,
|
| 123 |
+
"total": 872,
|
| 124 |
+
"accuracy": 0.5011467889908257,
|
| 125 |
+
"loss": 4.14792537689209,
|
| 126 |
+
"gold_label_counts": [
|
| 127 |
+
428,
|
| 128 |
+
444
|
| 129 |
+
],
|
| 130 |
+
"majority_accuracy": 0.5091743119266054
|
| 131 |
+
},
|
| 132 |
+
"after": {
|
| 133 |
+
"correct": 786,
|
| 134 |
+
"total": 872,
|
| 135 |
+
"accuracy": 0.9013761467889908,
|
| 136 |
+
"loss": 0.29649588465690613,
|
| 137 |
+
"gold_label_counts": [
|
| 138 |
+
428,
|
| 139 |
+
444
|
| 140 |
+
],
|
| 141 |
+
"majority_accuracy": 0.5091743119266054
|
| 142 |
+
},
|
| 143 |
+
"confusion_matrix": {
|
| 144 |
+
"before": [
|
| 145 |
+
[
|
| 146 |
+
398,
|
| 147 |
+
30
|
| 148 |
+
],
|
| 149 |
+
[
|
| 150 |
+
405,
|
| 151 |
+
39
|
| 152 |
+
]
|
| 153 |
+
],
|
| 154 |
+
"after": [
|
| 155 |
+
[
|
| 156 |
+
385,
|
| 157 |
+
43
|
| 158 |
+
],
|
| 159 |
+
[
|
| 160 |
+
43,
|
| 161 |
+
401
|
| 162 |
+
]
|
| 163 |
+
]
|
| 164 |
+
},
|
| 165 |
+
"train_examples": 67349,
|
| 166 |
+
"validation_examples": 872,
|
| 167 |
+
"test_examples": 1821
|
| 168 |
+
},
|
| 169 |
+
"provenance": {
|
| 170 |
+
"source_training_checkpoint": "best.pt",
|
| 171 |
+
"source_training_checkpoint_sha256": "2e2630c24231098e9381b09b2e6f3d27a273696743bfe65bffe4d5f2ecd4d80e",
|
| 172 |
+
"source_hashes": {
|
| 173 |
+
"model": "a28201b8194941a75d3ca2e0812e6d2d0eef1b2656cc0c9f8fd0ca65bf96b28b",
|
| 174 |
+
"trainer": "550f68528e3dbe630cb7f0ad29860d8bafb77767832819dab86ea598a29ad520"
|
| 175 |
+
},
|
| 176 |
+
"original_pretrained_checkpoint_sha256": "88afaab6c93ef3e33940b4e25cbce2f44038c428109600afe8285586a3a767b9",
|
| 177 |
+
"original_backbone_unchanged_during_export": true,
|
| 178 |
+
"task_weights": "independent full-model task adaptation of original pretrained backbone",
|
| 179 |
+
"export": "CPU state-dict serialization only; optimizer, order and RNG removed; no model construction or forward pass",
|
| 180 |
+
"torch_version": "2.11.0.dev20260215+cu128"
|
| 181 |
+
},
|
| 182 |
+
"file": "model.pt",
|
| 183 |
+
"bytes": 494839923,
|
| 184 |
+
"sha256": "0a0c73b9ec95da128c07abe6949abd2db8492c829f52bd7bb350f67aef684c66",
|
| 185 |
+
"validation": "All model tensors exactly equal to the completed best checkpoint; source SHA-256 matches comparison record."
|
| 186 |
+
}
|
task_adaptations/sst2/model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0a0c73b9ec95da128c07abe6949abd2db8492c829f52bd7bb350f67aef684c66
|
| 3 |
+
size 494839923
|
task_finetuning/.gitignore
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.pyc
|
| 3 |
+
data/*
|
| 4 |
+
!data/.gitkeep
|
| 5 |
+
runs/
|
| 6 |
+
checkpoints/
|
| 7 |
+
*.pending.pt
|
| 8 |
+
*.partial
|
task_finetuning/README.md
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# WiggleGPT: supervised task adaptation, 26 October 2026
|
| 2 |
+
|
| 3 |
+
Independent full-model adaptations of the original 600,000-step WiggleGPT checkpoint reached **90.1% on SST-2 validation** and **59.5% on RACE-middle test** after three complete epochs per task.
|
| 4 |
+
|
| 5 |
+
| Task and scored split | Before task training | After task training | Majority baseline |
|
| 6 |
+
|---|---:|---:|---:|
|
| 7 |
+
| SST-2 sentiment, validation | 437/872 (50.1%) | **786/872 (90.1%)** | 444/872 (50.9%) |
|
| 8 |
+
| RACE-middle reading, test | 385/1,436 (26.8%) | **854/1,436 (59.5%)** | 388/1,436 (27.0%) |
|
| 9 |
+
|
| 10 |
+
“Before” is the original pretrained backbone with the same newly initialized task classifier and task-marker embeddings, evaluated before the first task-training update. These are classifier decisions, rather than free-text answers. The two adaptations start independently from pretraining; neither starts from the SmolTalk instruction checkpoint. Every backbone parameter trains, including the oscillator parameters.
|
| 11 |
+
|
| 12 |
+
The [eight-page demonstration PDF](WiggleGPT_Demonstration.pdf) presents the results and examples. [results/comparison.json](results/comparison.json) contains all 2,308 paired held-out predictions, labels, source-text hashes, confusion matrices, validation histories and checkpoint hashes. It retains only the short excerpts selected for the demonstration; the complete benchmark corpus is obtained from its original sources.
|
| 13 |
+
|
| 14 |
+
## Training and data
|
| 15 |
+
|
| 16 |
+
| Setting | SST-2 | RACE-middle |
|
| 17 |
+
|---|---:|---:|
|
| 18 |
+
| Training examples | 67,349 | 25,421 |
|
| 19 |
+
| Validation examples | 872 | 1,436 |
|
| 20 |
+
| Scored held-out examples | 872 validation | 1,436 test |
|
| 21 |
+
| Complete epochs | 3 | 3 |
|
| 22 |
+
| Optimizer updates | 6,315 | 2,385 |
|
| 23 |
+
| Validation-selected best update | 6,315 | 2,385 |
|
| 24 |
+
| Effective batch | 32 reviews | 32 questions |
|
| 25 |
+
| Microbatch / accumulation | 16 / 2 | 2 questions / 16 |
|
| 26 |
+
| Peak allocated GPU memory | 3.846 GiB | 6.383 GiB |
|
| 27 |
+
|
| 28 |
+
Both runs stopped at `epochs_complete` with zero non-finite updates. The loss is task cross-entropy plus 0.5 times auxiliary language-model cross-entropy, averaged within each candidate sequence and then across candidates. Incorrect answer candidates also receive the auxiliary objective. Both use AdamW, peak learning rate 6.25e-5, 0.2% linear warmup followed by linear decay, classifier dropout 0.1, gradient clipping 1.0 and a maximum sequence length of 512. The pretrained backbone's dropout remains 0.0. Oscillator, one-dimensional and bias parameters have zero weight decay.
|
| 29 |
+
|
| 30 |
+
SST-2 uses `[START] review [CLASSIFY]` and a 768-to-2 classifier. RACE uses `[START] article [DELIMITER] question [DELIMITER] answer [CLASSIFY]` for each of four candidates, scored by one shared 768-to-1 classifier. Candidate feature dropout shares a mask across the four answers. The start, delimiter and classify markers occupy unused padded-vocabulary IDs 50257, 50258 and 50259; original GPT-2 token IDs remain unchanged. Right padding uses 50256. RACE article prefixes are budgeted against the longest answer so every candidate sees the same passage; questions and options remain complete. There were 210 truncated training passages, nine validation passages and 14 test passages; SST-2 had no truncation.
|
| 31 |
+
|
| 32 |
+
SST-2 validation supplies both checkpoint selection and the reported score because public test labels are withheld. RACE uses validation for selection and its separate test split for the final score. The historical GPT-1 paper reports 91.3% on SST-2 **GLUE test** and 62.9% on RACE-M after **combined middle/high training**. These provide context: WiggleGPT's SST-2 score is validation, and its RACE adaptation uses middle-only training.
|
| 33 |
+
|
| 34 |
+
## Run from this repository
|
| 35 |
+
|
| 36 |
+
Use Python 3.11+ and a CUDA-enabled PyTorch build. CPU preparation uses two threads; model construction, training and evaluation require CUDA. Run one GPU job at a time. The saved configuration caps the process at 9 GiB and reserves desktop memory.
|
| 37 |
+
|
| 38 |
+
```bash
|
| 39 |
+
pip install -r task_finetuning/requirements.txt
|
| 40 |
+
python task_finetuning/prepare_data.py
|
| 41 |
+
python task_finetuning/encode_task_data.py
|
| 42 |
+
```
|
| 43 |
+
|
| 44 |
+
The preparation script downloads the official SST-2 and RACE-middle splits. The bundled GPT-2/r50k merge table is checked against its SHA-256 before encoding. Generated data stays under `task_finetuning/data/` and is excluded from version control. The source manifests and encoding summaries from the completed experiment are under `results/`.
|
| 45 |
+
|
| 46 |
+
For a new adaptation, place the original pretrained `ckpt.pt` at `checkpoints/pretrained/ckpt.pt` in the repository root, or supply its path explicitly. It must be the original 600,000-step pretraining checkpoint, not a SmolTalk or task checkpoint.
|
| 47 |
+
|
| 48 |
+
```bash
|
| 49 |
+
python task_finetuning/train_tasks.py --task sst2 --starting-checkpoint checkpoints/pretrained/ckpt.pt
|
| 50 |
+
python task_finetuning/train_tasks_v3.py --task race_middle --starting-checkpoint checkpoints/pretrained/ckpt.pt
|
| 51 |
+
```
|
| 52 |
+
|
| 53 |
+
These commands run sequentially. The original trainer is retained for SST-2; version 3 is the final RACE trainer. Version 3 crops padding with a CPU epoch plan, projects only valid auxiliary-LM positions and enables native gradient checkpointing only above 384 cropped tokens. The completed RACE run began with version 2 and continued with version 3 from a saved update; source snapshots and provenance are included under `results/`. These changes preserve question order, batch membership and the per-candidate loss objective. A newly started version-3 run has the same recipe but need not be bit-identical to that segmented run.
|
| 54 |
+
|
| 55 |
+
Runs write `latest.pt`, validation-selected `best.pt`, their configuration, source snapshots and logs under `task_finetuning/runs/`. Resume with the appropriate trainer and a saved `latest.pt`:
|
| 56 |
+
|
| 57 |
+
```bash
|
| 58 |
+
python task_finetuning/train_tasks_v3.py --resume task_finetuning/runs/YOUR_RACE_RUN/latest.pt
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
The saved run configuration controls resume. Keep the original pretraining checkpoint at the path in that configuration. A single interrupt requests a save after the current update; on Windows, leave the process open until the save finishes. If a terminal terminates the process directly, resume from the last completed `latest.pt`.
|
| 62 |
+
|
| 63 |
+
## Evaluate released task weights
|
| 64 |
+
|
| 65 |
+
Task checkpoints are separate from the original language-model checkpoint. Download the released inference weights from the [Hugging Face model page](https://huggingface.co/edeneldith/WiggleGPT) and place the files in the paths below, or pass their actual paths. GitHub source archives do not contain the large weights.
|
| 66 |
+
|
| 67 |
+
```bash
|
| 68 |
+
python task_finetuning/evaluate_tasks.py --checkpoint task_adaptations/sst2/model.pt --output task_finetuning/results/sst2_recomputed.json
|
| 69 |
+
python task_finetuning/evaluate_tasks.py --checkpoint task_adaptations/race_middle/model.pt --output task_finetuning/results/race_middle_recomputed.json
|
| 70 |
+
```
|
| 71 |
+
|
| 72 |
+
The evaluator accepts both inference-only exports (`wigglegpt_task_inference_v1`) and full training `best.pt` checkpoints. It loads the native backbone and task head strictly, then scores SST-2 validation or RACE-middle test by default. `--split validation` selects validation explicitly. Scoring the hidden-label SST-2 test split is refused. Evaluation needs the prepared token caches and the task weights; the original pretrained checkpoint and optimizer state are unnecessary.
|
| 73 |
+
|
| 74 |
+
## Files and provenance
|
| 75 |
+
|
| 76 |
+
The existing root `model_bio.py` and pretraining scripts are unchanged. `native/model_bio.py` is the exact byte-preserved source used for the completed adaptations (SHA-256 `a28201b8194941a75d3ca2e0812e6d2d0eef1b2656cc0c9f8fd0ca65bf96b28b`). The downloaded repository's root model source differs only in comments/docstrings; the dedicated snapshot keeps provenance exact. `task_runtime.py` carries the original loader and exact tokenizer into this portable folder. The distributed training scripts adapt paths and command-line checkpoint selection; original run source snapshots remain in `results/` for reference.
|
| 77 |
+
|
| 78 |
+
The repository integration was checked for Python syntax, expected file references, source preservation and consistency with the saved prediction counts. No training or evaluation was rerun during packaging. `results/manifest.json` records SHA-256 hashes of this release folder.
|
| 79 |
+
|
| 80 |
+
Sources: [SST-2](https://huggingface.co/datasets/stanfordnlp/sst2), [RACE](https://huggingface.co/datasets/ehovy/race), [GPT-1 paper](https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf), [OpenAI's 2018 announcement](https://openai.com/index/language-unsupervised/), [tiktoken](https://github.com/openai/tiktoken).
|
| 81 |
+
|
| 82 |
+
P.C. O'Brien (ORCID 0009-0007-3961-1182), Independent Researcher, originated and directed WiggleGPT and this task extension. OpenAI Codex assisted with implementation, analysis and release preparation.
|
task_finetuning/WiggleGPT_Demonstration.pdf
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:032920d6249b0e392012bfb6da0f8e5d34ca1bb173f2c75b2bb902e580330ad1
|
| 3 |
+
size 2187032
|
task_finetuning/data/.gitkeep
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
|
task_finetuning/encode_task_data.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CPU-only exact GPT-2 token caches for GPT-1-style supervised adaptation."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import hashlib
|
| 5 |
+
import json
|
| 6 |
+
import math
|
| 7 |
+
import os
|
| 8 |
+
from datetime import datetime, timezone
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
os.environ.setdefault("OMP_NUM_THREADS", "2")
|
| 12 |
+
os.environ.setdefault("MKL_NUM_THREADS", "2")
|
| 13 |
+
import numpy as np
|
| 14 |
+
import torch
|
| 15 |
+
|
| 16 |
+
ROOT = Path(__file__).resolve().parent
|
| 17 |
+
DATA = ROOT / "data"
|
| 18 |
+
TOKENIZER_SOURCE = ROOT / "task_runtime.py"
|
| 19 |
+
from task_runtime import get_encoder
|
| 20 |
+
START, DELIM, CLS, PAD = 50257, 50258, 50259, 50256
|
| 21 |
+
LIMIT = 512
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def digest(path: Path) -> str:
|
| 25 |
+
value = hashlib.sha256()
|
| 26 |
+
with path.open("rb") as stream:
|
| 27 |
+
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
| 28 |
+
value.update(block)
|
| 29 |
+
return value.hexdigest()
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def encode_unique(encoder, strings: list[str]) -> dict[str, list[int]]:
|
| 33 |
+
unique = list(dict.fromkeys(strings))
|
| 34 |
+
encoded = encoder.encode_ordinary_batch(unique, num_threads=2)
|
| 35 |
+
return dict(zip(unique, encoded))
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def encode_split(encoder, task: str, split: str) -> dict:
|
| 39 |
+
source = DATA / f"{task}_{split}.jsonl"
|
| 40 |
+
rows = [json.loads(line) for line in source.read_text(encoding="utf-8").splitlines()]
|
| 41 |
+
labels = torch.tensor([row["label"] for row in rows], dtype=torch.long)
|
| 42 |
+
counts = {"examples_with_truncation": 0, "candidate_sequences_with_truncation": 0,
|
| 43 |
+
"total_tokens_removed": 0, "max_tokens_removed_per_example": 0}
|
| 44 |
+
if task == "sst2":
|
| 45 |
+
encoded = encoder.encode_ordinary_batch([row["text"] for row in rows], num_threads=2)
|
| 46 |
+
max_raw = max(map(len, encoded)) + 2
|
| 47 |
+
lengths = np.array([[min(len(tokens), LIMIT - 2) + 2] for tokens in encoded], dtype=np.int64)
|
| 48 |
+
pad_length = max(16, math.ceil(int(lengths.max()) / 16) * 16)
|
| 49 |
+
input_ids = np.full((len(rows), 1, pad_length), PAD, dtype=np.int32)
|
| 50 |
+
for index, tokens in enumerate(encoded):
|
| 51 |
+
removed = max(0, len(tokens) - (LIMIT - 2))
|
| 52 |
+
if removed:
|
| 53 |
+
counts["examples_with_truncation"] += 1
|
| 54 |
+
counts["candidate_sequences_with_truncation"] += 1
|
| 55 |
+
counts["total_tokens_removed"] += removed
|
| 56 |
+
counts["max_tokens_removed_per_example"] = max(counts["max_tokens_removed_per_example"], removed)
|
| 57 |
+
sequence = [START] + tokens[:LIMIT - 2] + [CLS]
|
| 58 |
+
input_ids[index, 0, :len(sequence)] = sequence
|
| 59 |
+
context_policy = "[START] + review first 510 tokens + [CLS]; right-pad to the split maximum rounded up to a multiple of 16 (minimum 16)."
|
| 60 |
+
else:
|
| 61 |
+
article_ids = encode_unique(encoder, [row["article"] for row in rows])
|
| 62 |
+
question_ids = encode_unique(encoder, [row["question"] for row in rows])
|
| 63 |
+
answer_ids = encode_unique(encoder, [answer for row in rows for answer in row["options"]])
|
| 64 |
+
pad_length = LIMIT
|
| 65 |
+
input_ids = np.full((len(rows), 4, pad_length), PAD, dtype=np.int32)
|
| 66 |
+
lengths = np.empty((len(rows), 4), dtype=np.int64)
|
| 67 |
+
max_raw = 0
|
| 68 |
+
for index, row in enumerate(rows):
|
| 69 |
+
article = article_ids[row["article"]]
|
| 70 |
+
question = question_ids[row["question"]]
|
| 71 |
+
answers = [answer_ids[answer] for answer in row["options"]]
|
| 72 |
+
overhead = 4 + len(question) + max(map(len, answers))
|
| 73 |
+
if overhead > LIMIT:
|
| 74 |
+
raise ValueError(f"RACE {split}/{index} {row['example_id']}: question and longest answer plus 4 markers need {overhead} tokens, beyond {LIMIT}.")
|
| 75 |
+
budget = LIMIT - overhead
|
| 76 |
+
shared_article = article[:budget]
|
| 77 |
+
removed = max(0, len(article) - budget)
|
| 78 |
+
if removed:
|
| 79 |
+
counts["examples_with_truncation"] += 1
|
| 80 |
+
counts["candidate_sequences_with_truncation"] += 4
|
| 81 |
+
counts["total_tokens_removed"] += removed
|
| 82 |
+
counts["max_tokens_removed_per_example"] = max(counts["max_tokens_removed_per_example"], removed)
|
| 83 |
+
for choice, answer in enumerate(answers):
|
| 84 |
+
sequence = [START] + shared_article + [DELIM] + question + [DELIM] + answer + [CLS]
|
| 85 |
+
lengths[index, choice] = len(sequence)
|
| 86 |
+
input_ids[index, choice, :len(sequence)] = sequence
|
| 87 |
+
max_raw = max(max_raw, 4 + len(article) + len(question) + len(answer))
|
| 88 |
+
context_policy = "[START] + shared article prefix + [DELIM] + complete question + [DELIM] + complete answer + [CLS]; all four choices retain the same article prefix, budgeted using the longest answer; right-pad to 512."
|
| 89 |
+
summary = {"task": task, "split": split, "examples": len(rows), "choices": int(input_ids.shape[1]),
|
| 90 |
+
"source_jsonl": str(source), "source_sha256": digest(source),
|
| 91 |
+
"pad_length": pad_length, "max_sequence_length": int(lengths.max()),
|
| 92 |
+
"min_sequence_length": int(lengths.min()), "max_untruncated_length": max_raw,
|
| 93 |
+
"truncation": counts, "context_policy": context_policy,
|
| 94 |
+
"input_ids_dtype": "int32", "lengths_dtype": "int64", "labels_dtype": "int64"}
|
| 95 |
+
output = DATA / f"{task}_{split}.pt"
|
| 96 |
+
temporary = output.with_suffix(".pt.partial")
|
| 97 |
+
torch.save({"input_ids": torch.from_numpy(input_ids), "lengths": torch.from_numpy(lengths), "labels": labels,
|
| 98 |
+
"example_ids": [row["example_id"] for row in rows], "metadata": summary,
|
| 99 |
+
"source_sha256": summary["source_sha256"]}, temporary)
|
| 100 |
+
temporary.replace(output)
|
| 101 |
+
summary.update({"cache_path": str(output), "cache_sha256": digest(output), "cache_bytes": output.stat().st_size})
|
| 102 |
+
print(f"{task}/{split}: shape {tuple(input_ids.shape)}, max length {lengths.max()}, truncated {counts['examples_with_truncation']:,} examples", flush=True)
|
| 103 |
+
return summary
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def main() -> None:
|
| 107 |
+
torch.set_num_threads(2)
|
| 108 |
+
torch.set_num_interop_threads(2)
|
| 109 |
+
encoder = get_encoder()
|
| 110 |
+
summary = {"prepared_at_utc": datetime.now(timezone.utc).isoformat(),
|
| 111 |
+
"encoder": "Exact bundled GPT-2/r50k via task_runtime.get_encoder; ordinary-string encoding only",
|
| 112 |
+
"tokenizer_source": str(TOKENIZER_SOURCE), "tokenizer_source_sha256": digest(TOKENIZER_SOURCE),
|
| 113 |
+
"merge_table_sha256": digest(ROOT / "tokenizer" / "gpt2.tiktoken"),
|
| 114 |
+
"task_markers": {"START": START, "DELIM": DELIM, "CLS": CLS, "PAD": PAD},
|
| 115 |
+
"limit": LIMIT, "cpu_threads": 2, "splits": {}}
|
| 116 |
+
for task in ("sst2", "race_middle"):
|
| 117 |
+
for split in ("train", "validation", "test"):
|
| 118 |
+
summary["splits"][f"{task}_{split}"] = encode_split(encoder, task, split)
|
| 119 |
+
path = DATA / "encoding_summary.json"
|
| 120 |
+
path.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
| 121 |
+
print(f"Encoding summary: {path}", flush=True)
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
if __name__ == "__main__":
|
| 125 |
+
main()
|
task_finetuning/evaluate_tasks.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Evaluate a trained task checkpoint on its labelled public held-out split.
|
| 2 |
+
|
| 3 |
+
Accepts either a complete training checkpoint or an inference-only release.
|
| 4 |
+
No optimizer or original pretrained checkpoint is needed for evaluation.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import argparse
|
| 9 |
+
import hashlib
|
| 10 |
+
import json
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
import sys
|
| 13 |
+
|
| 14 |
+
ROOT = Path(__file__).resolve().parent
|
| 15 |
+
sys.path.insert(0, str(ROOT / "native"))
|
| 16 |
+
import torch
|
| 17 |
+
from model_bio import GPT, GPTConfig
|
| 18 |
+
from task_runtime import check_gpu
|
| 19 |
+
from train_tasks import TaskModel, evaluate
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def main():
|
| 23 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 24 |
+
parser.add_argument("--checkpoint", type=Path, required=True)
|
| 25 |
+
parser.add_argument("--split", choices=["validation", "test"])
|
| 26 |
+
parser.add_argument("--batch-size", type=int)
|
| 27 |
+
parser.add_argument("--max-process-gib", type=float, default=9.0)
|
| 28 |
+
parser.add_argument("--output", type=Path, help="Optional JSON with scores and every prediction.")
|
| 29 |
+
args = parser.parse_args()
|
| 30 |
+
if args.max_process_gib <= 0 or args.batch_size is not None and args.batch_size <= 0:
|
| 31 |
+
parser.error("Memory cap and batch size must be positive.")
|
| 32 |
+
if not torch.cuda.is_available():
|
| 33 |
+
raise RuntimeError("CUDA required; CPU model execution is disabled.")
|
| 34 |
+
check_gpu()
|
| 35 |
+
torch.set_num_threads(2)
|
| 36 |
+
torch.cuda.set_device(0)
|
| 37 |
+
free, total = torch.cuda.mem_get_info()
|
| 38 |
+
budget = min(int(args.max_process_gib * 1024**3), total - int(2.5 * 1024**3),
|
| 39 |
+
free - int(0.5 * 1024**3))
|
| 40 |
+
if budget < 2 * 1024**3:
|
| 41 |
+
raise RuntimeError("Less than 2 GiB is available after reserving desktop memory.")
|
| 42 |
+
torch.cuda.set_per_process_memory_fraction(budget / total, 0)
|
| 43 |
+
torch.set_float32_matmul_precision("high")
|
| 44 |
+
saved = torch.load(args.checkpoint, map_location="cpu", weights_only=False, mmap=True)
|
| 45 |
+
task = saved["task"]
|
| 46 |
+
if task not in ("sst2", "race_middle"):
|
| 47 |
+
raise ValueError(f"Unknown task: {task}")
|
| 48 |
+
split = args.split or ("validation" if task == "sst2" else "test")
|
| 49 |
+
if task == "sst2" and split == "test":
|
| 50 |
+
raise ValueError("SST-2 public test labels are hidden. Use validation for labelled evaluation.")
|
| 51 |
+
source_hash = saved.get("source_hashes", saved.get("provenance", {}).get("source_hashes", {})).get("model")
|
| 52 |
+
if source_hash:
|
| 53 |
+
actual_hash = hashlib.sha256((ROOT / "native" / "model_bio.py").read_bytes()).hexdigest()
|
| 54 |
+
if source_hash != actual_hash:
|
| 55 |
+
raise ValueError("Native model source differs from the checkpoint's recorded source.")
|
| 56 |
+
config = GPTConfig(**saved["model_args"])
|
| 57 |
+
config.gradient_checkpointing = False
|
| 58 |
+
dropout = saved.get("config", {}).get("classifier_dropout", 0.1)
|
| 59 |
+
with torch.device("cuda:0"):
|
| 60 |
+
base = GPT(config)
|
| 61 |
+
model = TaskModel(base, task, dropout)
|
| 62 |
+
model.load_state_dict(saved["model"], strict=True)
|
| 63 |
+
del saved
|
| 64 |
+
cache_path = ROOT / "data" / f"{task}_{split}.pt"
|
| 65 |
+
cache = torch.load(cache_path, map_location="cpu", weights_only=False, mmap=True)
|
| 66 |
+
data = {key: cache[key].to("cuda:0") for key in ("input_ids", "lengths", "labels")}
|
| 67 |
+
if torch.any(data["labels"] < 0):
|
| 68 |
+
raise ValueError("This split has withheld labels and cannot be scored.")
|
| 69 |
+
result = evaluate(model, data, args.batch_size or (16 if task == "sst2" else 4))
|
| 70 |
+
result.update(task=task, split=split, checkpoint=str(args.checkpoint),
|
| 71 |
+
example_ids=cache.get("example_ids"), source_sha256=cache.get("source_sha256"),
|
| 72 |
+
device=torch.cuda.get_device_name(0))
|
| 73 |
+
print(json.dumps({key: value for key, value in result.items()
|
| 74 |
+
if key not in ("predictions", "example_ids")}, indent=2))
|
| 75 |
+
if args.output:
|
| 76 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 77 |
+
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
if __name__ == "__main__":
|
| 81 |
+
main()
|
task_finetuning/native/model_bio.py
ADDED
|
@@ -0,0 +1,536 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
WiggleGPT v3 - model_bio.py
|
| 3 |
+
===========================
|
| 4 |
+
|
| 5 |
+
Optimized GPT with Bio-Inspired Neurons - Training Speed Improvements
|
| 6 |
+
=====================================================================
|
| 7 |
+
Extended from model_bio.py with modern training optimizations while
|
| 8 |
+
preserving the core WiggleGPT architecture.
|
| 9 |
+
|
| 10 |
+
New optimizations:
|
| 11 |
+
1. RMSNorm - Faster alternative to LayerNorm (~7% speedup, simpler computation)
|
| 12 |
+
2. Rotary Position Embeddings (RoPE) - More efficient than learned embeddings
|
| 13 |
+
3. Gradient checkpointing support - Trade compute for memory
|
| 14 |
+
4. Better initialization - GPT-NeoX style depth scaling
|
| 15 |
+
5. Flash Attention 2 hints - Better kernel selection
|
| 16 |
+
|
| 17 |
+
These optimizations preserve the core model config (n_layer, n_head, n_embd, etc.)
|
| 18 |
+
and bio-inspired features (oscillating activations, sparse events).
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
import math
|
| 22 |
+
import inspect
|
| 23 |
+
from dataclasses import dataclass
|
| 24 |
+
|
| 25 |
+
import torch
|
| 26 |
+
import torch.nn as nn
|
| 27 |
+
from torch.nn import functional as F
|
| 28 |
+
from torch.utils.checkpoint import checkpoint
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
# ==============================================================================
|
| 32 |
+
# OPTIMIZED COMPONENTS
|
| 33 |
+
# ==============================================================================
|
| 34 |
+
|
| 35 |
+
class RMSNorm(nn.Module):
|
| 36 |
+
"""
|
| 37 |
+
Root Mean Square Layer Normalization
|
| 38 |
+
|
| 39 |
+
Simpler and faster than LayerNorm:
|
| 40 |
+
- No mean subtraction (only RMS normalization)
|
| 41 |
+
- No bias parameter
|
| 42 |
+
- ~7% faster than LayerNorm
|
| 43 |
+
- Used in LLaMA, GPT-NeoX, etc.
|
| 44 |
+
|
| 45 |
+
Formula: x * (gamma / rms(x)) where rms(x) = sqrt(mean(x^2) + eps)
|
| 46 |
+
"""
|
| 47 |
+
def __init__(self, dim: int, eps: float = 1e-6):
|
| 48 |
+
super().__init__()
|
| 49 |
+
self.eps = eps
|
| 50 |
+
self.weight = nn.Parameter(torch.ones(dim))
|
| 51 |
+
|
| 52 |
+
def forward(self, x):
|
| 53 |
+
rms = torch.sqrt(torch.mean(x ** 2, dim=-1, keepdim=True) + self.eps)
|
| 54 |
+
return x / rms * self.weight
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0):
|
| 58 |
+
"""
|
| 59 |
+
Precompute rotary position embedding frequencies.
|
| 60 |
+
|
| 61 |
+
Used for RoPE (Rotary Position Embeddings) - more efficient than learned
|
| 62 |
+
positional embeddings and enables better length extrapolation.
|
| 63 |
+
"""
|
| 64 |
+
freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim))
|
| 65 |
+
t = torch.arange(end, device=freqs.device)
|
| 66 |
+
freqs = torch.outer(t, freqs).float()
|
| 67 |
+
freqs_cis = torch.polar(torch.ones_like(freqs), freqs) # complex64
|
| 68 |
+
return freqs_cis
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def apply_rotary_emb(xq: torch.Tensor, xk: torch.Tensor, freqs_cis: torch.Tensor):
|
| 72 |
+
"""
|
| 73 |
+
Apply rotary position embeddings to query and key tensors.
|
| 74 |
+
|
| 75 |
+
RoPE encodes position information through rotation in complex space,
|
| 76 |
+
which naturally handles relative positions and enables length extrapolation.
|
| 77 |
+
"""
|
| 78 |
+
xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2))
|
| 79 |
+
xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2))
|
| 80 |
+
|
| 81 |
+
freqs_cis = freqs_cis[:xq_.shape[1]] # Truncate to sequence length
|
| 82 |
+
freqs_cis = freqs_cis.view(1, xq_.shape[1], 1, xq_.shape[-1])
|
| 83 |
+
|
| 84 |
+
xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3)
|
| 85 |
+
xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3)
|
| 86 |
+
|
| 87 |
+
return xq_out.type_as(xq), xk_out.type_as(xk)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
# ==============================================================================
|
| 91 |
+
# BIO-INSPIRED COMPONENTS (unchanged from model_bio.py)
|
| 92 |
+
# ==============================================================================
|
| 93 |
+
|
| 94 |
+
class OscillatingActivation(nn.Module):
|
| 95 |
+
"""
|
| 96 |
+
Oscillating activation function: sin(omega*x + phi)*tanh(x) + baseline
|
| 97 |
+
|
| 98 |
+
Enables single neurons to learn XOR and other non-linearly separable functions.
|
| 99 |
+
Key research finding: biological neurons can solve XOR, artificial neurons cannot.
|
| 100 |
+
"""
|
| 101 |
+
def __init__(self, num_features: int, learnable: bool = True):
|
| 102 |
+
super().__init__()
|
| 103 |
+
if learnable:
|
| 104 |
+
self.omega = nn.Parameter(torch.randn(num_features) * 0.1 + 1.0)
|
| 105 |
+
self.phi = nn.Parameter(torch.randn(num_features) * 0.1)
|
| 106 |
+
self.baseline = nn.Parameter(torch.zeros(num_features))
|
| 107 |
+
else:
|
| 108 |
+
self.register_buffer('omega', torch.ones(num_features))
|
| 109 |
+
self.register_buffer('phi', torch.zeros(num_features))
|
| 110 |
+
self.register_buffer('baseline', torch.zeros(num_features))
|
| 111 |
+
|
| 112 |
+
def forward(self, x):
|
| 113 |
+
osc = torch.sin(self.omega * x + self.phi)
|
| 114 |
+
return osc * torch.tanh(x) + self.baseline
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
# SparseEventLayer removed - testing pure oscillating neurons only
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
class BioMLP(nn.Module):
|
| 121 |
+
"""
|
| 122 |
+
PURE Bio-inspired MLP: Just oscillating activations, nothing else.
|
| 123 |
+
|
| 124 |
+
Drop-in replacement for standard MLP in GPT architecture.
|
| 125 |
+
Same parameter count as baseline - only difference is the activation function.
|
| 126 |
+
|
| 127 |
+
Standard MLP: Linear -> GELU -> Linear
|
| 128 |
+
WiggleGPT MLP: Linear -> sin(omega*x + phi)*tanh(x) -> Linear
|
| 129 |
+
|
| 130 |
+
Tests the core hypothesis: Can oscillating neurons beat static activations?
|
| 131 |
+
"""
|
| 132 |
+
def __init__(self, config):
|
| 133 |
+
super().__init__()
|
| 134 |
+
|
| 135 |
+
# Standard transformer MLP structure
|
| 136 |
+
self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=config.bias)
|
| 137 |
+
|
| 138 |
+
# ONLY CHANGE: Replace GELU with oscillating activation
|
| 139 |
+
self.activation = OscillatingActivation(4 * config.n_embd, learnable=True)
|
| 140 |
+
|
| 141 |
+
self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=config.bias)
|
| 142 |
+
self.dropout = nn.Dropout(config.dropout)
|
| 143 |
+
|
| 144 |
+
def forward(self, x):
|
| 145 |
+
x = self.c_fc(x)
|
| 146 |
+
x = self.activation(x) # Pure oscillating, no gating, no complexity
|
| 147 |
+
x = self.c_proj(x)
|
| 148 |
+
x = self.dropout(x)
|
| 149 |
+
return x
|
| 150 |
+
|
| 151 |
+
def get_avg_sparsity(self):
|
| 152 |
+
# No sparsity in this version - testing pure oscillation
|
| 153 |
+
return None
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
# ==============================================================================
|
| 157 |
+
# OPTIMIZED NANOGPT COMPONENTS
|
| 158 |
+
# ==============================================================================
|
| 159 |
+
|
| 160 |
+
class CausalSelfAttention(nn.Module):
|
| 161 |
+
"""
|
| 162 |
+
Optimized multi-head causal self-attention with:
|
| 163 |
+
- Flash Attention support
|
| 164 |
+
- Optional RoPE (Rotary Position Embeddings)
|
| 165 |
+
- Better initialization hints
|
| 166 |
+
"""
|
| 167 |
+
def __init__(self, config):
|
| 168 |
+
super().__init__()
|
| 169 |
+
assert config.n_embd % config.n_head == 0
|
| 170 |
+
|
| 171 |
+
# key, query, value projections for all heads, but in a batch
|
| 172 |
+
self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd, bias=config.bias)
|
| 173 |
+
# output projection
|
| 174 |
+
self.c_proj = nn.Linear(config.n_embd, config.n_embd, bias=config.bias)
|
| 175 |
+
# regularization
|
| 176 |
+
self.attn_dropout = nn.Dropout(config.dropout)
|
| 177 |
+
self.resid_dropout = nn.Dropout(config.dropout)
|
| 178 |
+
self.n_head = config.n_head
|
| 179 |
+
self.n_embd = config.n_embd
|
| 180 |
+
self.dropout = config.dropout
|
| 181 |
+
self.use_rope = config.use_rope
|
| 182 |
+
|
| 183 |
+
# flash attention make GPU go brrrrr but support is only in PyTorch >= 2.0
|
| 184 |
+
self.flash = hasattr(torch.nn.functional, 'scaled_dot_product_attention')
|
| 185 |
+
if not self.flash:
|
| 186 |
+
print("WARNING: using slow attention. Flash Attention requires PyTorch >= 2.0")
|
| 187 |
+
# causal mask to ensure that attention is only applied to the left in the input sequence
|
| 188 |
+
self.register_buffer("bias", torch.tril(torch.ones(config.block_size, config.block_size))
|
| 189 |
+
.view(1, 1, config.block_size, config.block_size))
|
| 190 |
+
|
| 191 |
+
def forward(self, x, freqs_cis=None):
|
| 192 |
+
B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd)
|
| 193 |
+
|
| 194 |
+
# calculate query, key, values for all heads in batch and move head forward to be the batch dim
|
| 195 |
+
q, k, v = self.c_attn(x).split(self.n_embd, dim=2)
|
| 196 |
+
k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
|
| 197 |
+
q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
|
| 198 |
+
v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
|
| 199 |
+
|
| 200 |
+
# Apply rotary embeddings if enabled
|
| 201 |
+
if self.use_rope and freqs_cis is not None:
|
| 202 |
+
q, k = apply_rotary_emb(q, k, freqs_cis)
|
| 203 |
+
|
| 204 |
+
# causal self-attention
|
| 205 |
+
if self.flash:
|
| 206 |
+
# efficient attention using Flash Attention CUDA kernels
|
| 207 |
+
y = torch.nn.functional.scaled_dot_product_attention(
|
| 208 |
+
q, k, v,
|
| 209 |
+
attn_mask=None,
|
| 210 |
+
dropout_p=self.dropout if self.training else 0,
|
| 211 |
+
is_causal=True
|
| 212 |
+
)
|
| 213 |
+
else:
|
| 214 |
+
# manual implementation of attention
|
| 215 |
+
att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1)))
|
| 216 |
+
att = att.masked_fill(self.bias[:, :, :T, :T] == 0, float('-inf'))
|
| 217 |
+
att = F.softmax(att, dim=-1)
|
| 218 |
+
att = self.attn_dropout(att)
|
| 219 |
+
y = att @ v # (B, nh, T, T) x (B, nh, T, hs) -> (B, nh, T, hs)
|
| 220 |
+
|
| 221 |
+
y = y.transpose(1, 2).contiguous().view(B, T, C) # re-assemble all head outputs side by side
|
| 222 |
+
|
| 223 |
+
# output projection
|
| 224 |
+
y = self.resid_dropout(self.c_proj(y))
|
| 225 |
+
return y
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
class MLP(nn.Module):
|
| 229 |
+
"""Standard nanoGPT MLP"""
|
| 230 |
+
def __init__(self, config):
|
| 231 |
+
super().__init__()
|
| 232 |
+
self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=config.bias)
|
| 233 |
+
self.gelu = nn.GELU()
|
| 234 |
+
self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=config.bias)
|
| 235 |
+
self.dropout = nn.Dropout(config.dropout)
|
| 236 |
+
|
| 237 |
+
def forward(self, x):
|
| 238 |
+
x = self.c_fc(x)
|
| 239 |
+
x = self.gelu(x)
|
| 240 |
+
x = self.c_proj(x)
|
| 241 |
+
x = self.dropout(x)
|
| 242 |
+
return x
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
class Block(nn.Module):
|
| 246 |
+
"""
|
| 247 |
+
Transformer block with optional gradient checkpointing.
|
| 248 |
+
"""
|
| 249 |
+
def __init__(self, config):
|
| 250 |
+
super().__init__()
|
| 251 |
+
# Use RMSNorm if enabled, otherwise standard LayerNorm
|
| 252 |
+
norm_class = RMSNorm if config.use_rmsnorm else nn.LayerNorm
|
| 253 |
+
if config.use_rmsnorm:
|
| 254 |
+
self.ln_1 = norm_class(config.n_embd)
|
| 255 |
+
self.ln_2 = norm_class(config.n_embd)
|
| 256 |
+
else:
|
| 257 |
+
# Standard LayerNorm with bias option
|
| 258 |
+
self.ln_1 = nn.LayerNorm(config.n_embd, bias=config.bias)
|
| 259 |
+
self.ln_2 = nn.LayerNorm(config.n_embd, bias=config.bias)
|
| 260 |
+
|
| 261 |
+
self.attn = CausalSelfAttention(config)
|
| 262 |
+
|
| 263 |
+
# Choose MLP type based on config
|
| 264 |
+
if config.use_bio_mlp:
|
| 265 |
+
self.mlp = BioMLP(config)
|
| 266 |
+
else:
|
| 267 |
+
self.mlp = MLP(config)
|
| 268 |
+
|
| 269 |
+
def forward(self, x, freqs_cis=None):
|
| 270 |
+
x = x + self.attn(self.ln_1(x), freqs_cis)
|
| 271 |
+
x = x + self.mlp(self.ln_2(x))
|
| 272 |
+
return x
|
| 273 |
+
|
| 274 |
+
def get_avg_sparsity(self):
|
| 275 |
+
"""Get sparsity metric if using bio MLP"""
|
| 276 |
+
if hasattr(self.mlp, 'get_avg_sparsity'):
|
| 277 |
+
return self.mlp.get_avg_sparsity()
|
| 278 |
+
return None
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
@dataclass
|
| 282 |
+
class GPTConfig:
|
| 283 |
+
block_size: int = 1024
|
| 284 |
+
vocab_size: int = 50304 # GPT-2 vocab_size of 50257, padded up to nearest multiple of 64 for efficiency
|
| 285 |
+
n_layer: int = 12
|
| 286 |
+
n_head: int = 12
|
| 287 |
+
n_embd: int = 768
|
| 288 |
+
dropout: float = 0.0
|
| 289 |
+
bias: bool = True # True: bias in Linears and LayerNorms, like GPT-2. False: a bit better and faster
|
| 290 |
+
|
| 291 |
+
# Bio-inspired neuron parameters
|
| 292 |
+
use_bio_mlp: bool = False # Use oscillating neurons instead of GELU
|
| 293 |
+
|
| 294 |
+
# Optimization parameters
|
| 295 |
+
use_rmsnorm: bool = False # Use RMSNorm instead of LayerNorm (faster)
|
| 296 |
+
use_rope: bool = False # Use Rotary Position Embeddings instead of learned
|
| 297 |
+
gradient_checkpointing: bool = False # Trade compute for memory
|
| 298 |
+
|
| 299 |
+
|
| 300 |
+
class GPT(nn.Module):
|
| 301 |
+
def __init__(self, config):
|
| 302 |
+
super().__init__()
|
| 303 |
+
assert config.vocab_size is not None
|
| 304 |
+
assert config.block_size is not None
|
| 305 |
+
self.config = config
|
| 306 |
+
|
| 307 |
+
module_dict = {
|
| 308 |
+
'wte': nn.Embedding(config.vocab_size, config.n_embd),
|
| 309 |
+
'drop': nn.Dropout(config.dropout),
|
| 310 |
+
'h': nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
|
| 311 |
+
}
|
| 312 |
+
|
| 313 |
+
# Add positional embeddings if not using RoPE
|
| 314 |
+
if not config.use_rope:
|
| 315 |
+
module_dict['wpe'] = nn.Embedding(config.block_size, config.n_embd)
|
| 316 |
+
|
| 317 |
+
# Add final norm
|
| 318 |
+
if config.use_rmsnorm:
|
| 319 |
+
module_dict['ln_f'] = RMSNorm(config.n_embd)
|
| 320 |
+
else:
|
| 321 |
+
module_dict['ln_f'] = nn.LayerNorm(config.n_embd, bias=config.bias)
|
| 322 |
+
|
| 323 |
+
self.transformer = nn.ModuleDict(module_dict)
|
| 324 |
+
self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
|
| 325 |
+
|
| 326 |
+
# Weight tying
|
| 327 |
+
self.transformer.wte.weight = self.lm_head.weight # https://paperswithcode.com/method/weight-tying
|
| 328 |
+
|
| 329 |
+
# Precompute RoPE frequencies if using RoPE
|
| 330 |
+
if config.use_rope:
|
| 331 |
+
freqs_cis = precompute_freqs_cis(
|
| 332 |
+
config.n_embd // config.n_head,
|
| 333 |
+
config.block_size * 2 # Allow for some extrapolation
|
| 334 |
+
)
|
| 335 |
+
self.register_buffer("freqs_cis", freqs_cis, persistent=False)
|
| 336 |
+
|
| 337 |
+
# Init all weights
|
| 338 |
+
self.apply(self._init_weights)
|
| 339 |
+
|
| 340 |
+
# Apply special scaled init to the residual projections (GPT-NeoX style)
|
| 341 |
+
for pn, p in self.named_parameters():
|
| 342 |
+
if pn.endswith('c_proj.weight'):
|
| 343 |
+
# GPT-NeoX uses 2 * n_layer for better depth scaling
|
| 344 |
+
torch.nn.init.normal_(p, mean=0.0, std=0.02 / math.sqrt(2 * config.n_layer))
|
| 345 |
+
|
| 346 |
+
# Report number of parameters
|
| 347 |
+
print("number of parameters: %.2fM" % (self.get_num_params() / 1e6,))
|
| 348 |
+
|
| 349 |
+
def get_num_params(self, non_embedding=True):
|
| 350 |
+
"""
|
| 351 |
+
Return the number of parameters in the model.
|
| 352 |
+
For non-embedding count (default), the position embeddings get subtracted.
|
| 353 |
+
The token embeddings would too, except due to the parameter sharing these
|
| 354 |
+
params are actually used as weights in the final layer, so we include them.
|
| 355 |
+
"""
|
| 356 |
+
n_params = sum(p.numel() for p in self.parameters())
|
| 357 |
+
if non_embedding and not self.config.use_rope:
|
| 358 |
+
n_params -= self.transformer.wpe.weight.numel()
|
| 359 |
+
return n_params
|
| 360 |
+
|
| 361 |
+
def _init_weights(self, module):
|
| 362 |
+
if isinstance(module, nn.Linear):
|
| 363 |
+
torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
| 364 |
+
if module.bias is not None:
|
| 365 |
+
torch.nn.init.zeros_(module.bias)
|
| 366 |
+
elif isinstance(module, nn.Embedding):
|
| 367 |
+
torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
| 368 |
+
|
| 369 |
+
def forward(self, idx, targets=None):
|
| 370 |
+
device = idx.device
|
| 371 |
+
b, t = idx.size()
|
| 372 |
+
assert t <= self.config.block_size, f"Cannot forward sequence of length {t}, block size is only {self.config.block_size}"
|
| 373 |
+
|
| 374 |
+
# Forward the GPT model itself
|
| 375 |
+
tok_emb = self.transformer.wte(idx) # token embeddings of shape (b, t, n_embd)
|
| 376 |
+
|
| 377 |
+
if self.config.use_rope:
|
| 378 |
+
# RoPE: no position embeddings added, positions encoded in attention
|
| 379 |
+
x = self.transformer.drop(tok_emb)
|
| 380 |
+
freqs_cis = self.freqs_cis[:t]
|
| 381 |
+
else:
|
| 382 |
+
# Standard learned positional embeddings
|
| 383 |
+
pos = torch.arange(0, t, dtype=torch.long, device=device) # shape (t)
|
| 384 |
+
pos_emb = self.transformer.wpe(pos) # position embeddings of shape (t, n_embd)
|
| 385 |
+
x = self.transformer.drop(tok_emb + pos_emb)
|
| 386 |
+
freqs_cis = None
|
| 387 |
+
|
| 388 |
+
# Apply transformer blocks with optional gradient checkpointing
|
| 389 |
+
if self.config.gradient_checkpointing and self.training:
|
| 390 |
+
for block in self.transformer.h:
|
| 391 |
+
x = checkpoint(block, x, freqs_cis, use_reentrant=False)
|
| 392 |
+
else:
|
| 393 |
+
for block in self.transformer.h:
|
| 394 |
+
x = block(x, freqs_cis)
|
| 395 |
+
|
| 396 |
+
x = self.transformer.ln_f(x)
|
| 397 |
+
|
| 398 |
+
if targets is not None:
|
| 399 |
+
# if we are given some desired targets also calculate the loss
|
| 400 |
+
logits = self.lm_head(x)
|
| 401 |
+
loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)
|
| 402 |
+
else:
|
| 403 |
+
# inference-time mini-optimization: only forward the lm_head on the very last position
|
| 404 |
+
logits = self.lm_head(x[:, [-1], :]) # note: using list [-1] to preserve the time dim
|
| 405 |
+
loss = None
|
| 406 |
+
|
| 407 |
+
return logits, loss
|
| 408 |
+
|
| 409 |
+
def configure_optimizers(self, weight_decay, learning_rate, betas, device_type, optimizer_type='adamw'):
|
| 410 |
+
"""
|
| 411 |
+
Configure optimizer with support for multiple optimizer types.
|
| 412 |
+
|
| 413 |
+
Supported optimizers:
|
| 414 |
+
- 'adamw': Standard AdamW (fused if available on CUDA)
|
| 415 |
+
- 'adamw8bit': 8-bit AdamW from bitsandbytes (memory efficient)
|
| 416 |
+
- 'lion': Lion optimizer (faster convergence, lower memory)
|
| 417 |
+
- 'adafactor': Adafactor (memory efficient, no beta2)
|
| 418 |
+
"""
|
| 419 |
+
# Start with all of the candidate parameters
|
| 420 |
+
param_dict = {pn: p for pn, p in self.named_parameters()}
|
| 421 |
+
# Filter out those that do not require grad
|
| 422 |
+
param_dict = {pn: p for pn, p in param_dict.items() if p.requires_grad}
|
| 423 |
+
|
| 424 |
+
# Create optim groups. Any parameters that is 2D will be weight decayed, otherwise no.
|
| 425 |
+
# i.e. all weight tensors in matmuls + embeddings decay, all biases and layernorms don't.
|
| 426 |
+
decay_params = [p for n, p in param_dict.items() if p.dim() >= 2]
|
| 427 |
+
nodecay_params = [p for n, p in param_dict.items() if p.dim() < 2]
|
| 428 |
+
optim_groups = [
|
| 429 |
+
{'params': decay_params, 'weight_decay': weight_decay},
|
| 430 |
+
{'params': nodecay_params, 'weight_decay': 0.0}
|
| 431 |
+
]
|
| 432 |
+
num_decay_params = sum(p.numel() for p in decay_params)
|
| 433 |
+
num_nodecay_params = sum(p.numel() for p in nodecay_params)
|
| 434 |
+
print(f"num decayed parameter tensors: {len(decay_params)}, with {num_decay_params:,} parameters")
|
| 435 |
+
print(f"num non-decayed parameter tensors: {len(nodecay_params)}, with {num_nodecay_params:,} parameters")
|
| 436 |
+
|
| 437 |
+
if optimizer_type == 'adamw':
|
| 438 |
+
# Create AdamW optimizer and use the fused version if it is available
|
| 439 |
+
fused_available = 'fused' in inspect.signature(torch.optim.AdamW).parameters
|
| 440 |
+
use_fused = fused_available and device_type == 'cuda'
|
| 441 |
+
extra_args = dict(fused=True) if use_fused else dict()
|
| 442 |
+
optimizer = torch.optim.AdamW(optim_groups, lr=learning_rate, betas=betas, **extra_args)
|
| 443 |
+
print(f"using fused AdamW: {use_fused}")
|
| 444 |
+
|
| 445 |
+
elif optimizer_type == 'adamw8bit':
|
| 446 |
+
try:
|
| 447 |
+
import bitsandbytes as bnb
|
| 448 |
+
optimizer = bnb.optim.AdamW8bit(optim_groups, lr=learning_rate, betas=betas)
|
| 449 |
+
print("using 8-bit AdamW from bitsandbytes")
|
| 450 |
+
except ImportError:
|
| 451 |
+
print("WARNING: bitsandbytes not installed, falling back to standard AdamW")
|
| 452 |
+
print("Install with: pip install bitsandbytes")
|
| 453 |
+
optimizer = torch.optim.AdamW(optim_groups, lr=learning_rate, betas=betas)
|
| 454 |
+
|
| 455 |
+
elif optimizer_type == 'lion':
|
| 456 |
+
try:
|
| 457 |
+
from lion_pytorch import Lion
|
| 458 |
+
optimizer = Lion(optim_groups, lr=learning_rate, betas=betas)
|
| 459 |
+
print("using Lion optimizer")
|
| 460 |
+
except ImportError:
|
| 461 |
+
print("WARNING: lion-pytorch not installed, falling back to AdamW")
|
| 462 |
+
print("Install with: pip install lion-pytorch")
|
| 463 |
+
optimizer = torch.optim.AdamW(optim_groups, lr=learning_rate, betas=betas)
|
| 464 |
+
|
| 465 |
+
elif optimizer_type == 'adafactor':
|
| 466 |
+
try:
|
| 467 |
+
from transformers.optimization import Adafactor
|
| 468 |
+
# Adafactor doesn't use weight_decay in the same way, merge groups
|
| 469 |
+
all_params = decay_params + nodecay_params
|
| 470 |
+
optimizer = Adafactor(
|
| 471 |
+
all_params,
|
| 472 |
+
lr=learning_rate,
|
| 473 |
+
scale_parameter=False,
|
| 474 |
+
relative_step=False,
|
| 475 |
+
warmup_init=False
|
| 476 |
+
)
|
| 477 |
+
print("using Adafactor optimizer")
|
| 478 |
+
except ImportError:
|
| 479 |
+
print("WARNING: transformers not installed, falling back to AdamW")
|
| 480 |
+
print("Install with: pip install transformers")
|
| 481 |
+
optimizer = torch.optim.AdamW(optim_groups, lr=learning_rate, betas=betas)
|
| 482 |
+
|
| 483 |
+
else:
|
| 484 |
+
raise ValueError(f"Unknown optimizer type: {optimizer_type}")
|
| 485 |
+
|
| 486 |
+
return optimizer
|
| 487 |
+
|
| 488 |
+
def estimate_mfu(self, fwdbwd_per_iter, dt):
|
| 489 |
+
"""estimate model flops utilization (MFU) in units of A100 bfloat16 peak FLOPS"""
|
| 490 |
+
N = self.get_num_params()
|
| 491 |
+
cfg = self.config
|
| 492 |
+
L, H, Q, T = cfg.n_layer, cfg.n_head, cfg.n_embd // cfg.n_head, cfg.block_size
|
| 493 |
+
flops_per_token = 6 * N + 12 * L * H * Q * T
|
| 494 |
+
flops_per_fwdbwd = flops_per_token * T
|
| 495 |
+
flops_per_iter = flops_per_fwdbwd * fwdbwd_per_iter
|
| 496 |
+
# express our flops throughput as ratio of A100 bfloat16 peak flops
|
| 497 |
+
flops_achieved = flops_per_iter * (1.0 / dt) # per second
|
| 498 |
+
flops_promised = 312e12 # A100 GPU bfloat16 peak flops is 312 TFLOPS
|
| 499 |
+
mfu = flops_achieved / flops_promised
|
| 500 |
+
return mfu
|
| 501 |
+
|
| 502 |
+
@torch.no_grad()
|
| 503 |
+
def generate(self, idx, max_new_tokens, temperature=1.0, top_k=None):
|
| 504 |
+
"""
|
| 505 |
+
Take a conditioning sequence of indices idx (LongTensor of shape (b,t)) and complete
|
| 506 |
+
the sequence max_new_tokens times, feeding the predictions back into the model each time.
|
| 507 |
+
Most likely you'll want to make sure to be in model.eval() mode of operation for this.
|
| 508 |
+
"""
|
| 509 |
+
for _ in range(max_new_tokens):
|
| 510 |
+
# if the sequence context is growing too long we must crop it at block_size
|
| 511 |
+
idx_cond = idx if idx.size(1) <= self.config.block_size else idx[:, -self.config.block_size:]
|
| 512 |
+
# forward the model to get the logits for the index in the sequence
|
| 513 |
+
logits, _ = self(idx_cond)
|
| 514 |
+
# pluck the logits at the final step and scale by desired temperature
|
| 515 |
+
logits = logits[:, -1, :] / temperature
|
| 516 |
+
# optionally crop the logits to only the top k options
|
| 517 |
+
if top_k is not None:
|
| 518 |
+
v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
|
| 519 |
+
logits[logits < v[:, [-1]]] = -float('Inf')
|
| 520 |
+
# apply softmax to convert logits to (normalized) probabilities
|
| 521 |
+
probs = F.softmax(logits, dim=-1)
|
| 522 |
+
# sample from the distribution
|
| 523 |
+
idx_next = torch.multinomial(probs, num_samples=1)
|
| 524 |
+
# append sampled index to the running sequence and continue
|
| 525 |
+
idx = torch.cat((idx, idx_next), dim=1)
|
| 526 |
+
|
| 527 |
+
return idx
|
| 528 |
+
|
| 529 |
+
def get_avg_sparsity(self):
|
| 530 |
+
"""Get average sparsity across all blocks if using bio MLP"""
|
| 531 |
+
sparsities = []
|
| 532 |
+
for block in self.transformer.h:
|
| 533 |
+
s = block.get_avg_sparsity()
|
| 534 |
+
if s is not None:
|
| 535 |
+
sparsities.append(s)
|
| 536 |
+
return sum(sparsities) / len(sparsities) if sparsities else None
|
task_finetuning/prepare_data.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Download public benchmark parquets and preserve exact normalized task splits.
|
| 2 |
+
|
| 3 |
+
This script performs no model work. Public SST-2 test labels are withheld (-1);
|
| 4 |
+
the validation split is therefore the labelled held-out evaluation split.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import hashlib
|
| 9 |
+
import json
|
| 10 |
+
import time
|
| 11 |
+
import urllib.parse
|
| 12 |
+
import urllib.request
|
| 13 |
+
from collections import Counter
|
| 14 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 15 |
+
from datetime import datetime, timezone
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
|
| 18 |
+
import pyarrow as pa
|
| 19 |
+
import pyarrow.parquet as pq
|
| 20 |
+
|
| 21 |
+
ROOT = Path(__file__).resolve().parent / "data"
|
| 22 |
+
SOURCES = [("sst2", "stanfordnlp/sst2", "default"), ("race_middle", "ehovy/race", "middle")]
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def digest(path: Path) -> str:
|
| 26 |
+
value = hashlib.sha256()
|
| 27 |
+
with path.open("rb") as stream:
|
| 28 |
+
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
| 29 |
+
value.update(block)
|
| 30 |
+
return value.hexdigest()
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def fetch(url: str, path: Path) -> None:
|
| 34 |
+
if path.exists():
|
| 35 |
+
return
|
| 36 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 37 |
+
temporary = path.with_suffix(path.suffix + ".partial")
|
| 38 |
+
for attempt in range(4):
|
| 39 |
+
try:
|
| 40 |
+
request = urllib.request.Request(url, headers={"User-Agent": "WiggleGPT-benchmark-preparation/1.0"})
|
| 41 |
+
with urllib.request.urlopen(request, timeout=90) as response, temporary.open("wb") as stream:
|
| 42 |
+
while block := response.read(1024 * 1024):
|
| 43 |
+
stream.write(block)
|
| 44 |
+
temporary.replace(path)
|
| 45 |
+
return
|
| 46 |
+
except Exception:
|
| 47 |
+
if attempt == 3:
|
| 48 |
+
raise
|
| 49 |
+
time.sleep(2 ** attempt)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def normalize(task: str, row: dict, index: int, split: str) -> dict:
|
| 53 |
+
if task == "sst2":
|
| 54 |
+
return {"example_id": f"sst2/{split}/{row.get('idx', index)}", "text": row["sentence"], "label": int(row["label"])}
|
| 55 |
+
answer = row["answer"]
|
| 56 |
+
label = "ABCD".index(answer) if isinstance(answer, str) else int(answer)
|
| 57 |
+
options = list(row["options"])
|
| 58 |
+
if len(options) != 4 or not 0 <= label < 4:
|
| 59 |
+
raise ValueError(f"Invalid RACE record {index}")
|
| 60 |
+
return {"example_id": str(row.get("example_id", f"race_middle/{split}/{index}")),
|
| 61 |
+
"article": row["article"], "question": row["question"], "options": options, "label": label}
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def prepare_source(spec: tuple[str, str, str]) -> dict:
|
| 65 |
+
task, dataset, config = spec
|
| 66 |
+
metadata_url = "https://datasets-server.huggingface.co/parquet?" + urllib.parse.urlencode({"dataset": dataset})
|
| 67 |
+
metadata_path = ROOT / "source_parquets" / task / "parquet_metadata.json"
|
| 68 |
+
fetch(metadata_url, metadata_path)
|
| 69 |
+
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
| 70 |
+
if metadata.get("partial"):
|
| 71 |
+
raise RuntimeError(f"Partial parquet conversion for {dataset}")
|
| 72 |
+
files = [entry for entry in metadata["parquet_files"] if entry["config"] == config]
|
| 73 |
+
result = {"dataset": dataset, "config": config, "metadata_url": metadata_url, "splits": {}}
|
| 74 |
+
for split in ("train", "validation", "test"):
|
| 75 |
+
split_files = sorted([entry for entry in files if entry["split"] == split], key=lambda entry: entry["filename"])
|
| 76 |
+
if not split_files:
|
| 77 |
+
raise RuntimeError(f"Missing {dataset} {config} {split}")
|
| 78 |
+
destination = ROOT / f"{task}_{split}.jsonl"
|
| 79 |
+
temporary = destination.with_suffix(".jsonl.partial")
|
| 80 |
+
count, labels, source_files = 0, Counter(), []
|
| 81 |
+
with temporary.open("w", encoding="utf-8", newline="\n") as output:
|
| 82 |
+
for ordinal, entry in enumerate(split_files):
|
| 83 |
+
parquet_path = ROOT / "source_parquets" / task / split / f"{ordinal:03d}_{entry['filename']}"
|
| 84 |
+
fetch(entry["url"], parquet_path)
|
| 85 |
+
if parquet_path.stat().st_size != entry["size"]:
|
| 86 |
+
raise RuntimeError(f"Size mismatch for {parquet_path}")
|
| 87 |
+
source_files.append({"url": entry["url"], "path": str(parquet_path), "bytes": parquet_path.stat().st_size, "sha256": digest(parquet_path)})
|
| 88 |
+
for batch in pq.ParquetFile(parquet_path).iter_batches(batch_size=1024):
|
| 89 |
+
for row in batch.to_pylist():
|
| 90 |
+
item = normalize(task, row, count, split)
|
| 91 |
+
output.write(json.dumps(item, ensure_ascii=False) + "\n")
|
| 92 |
+
labels[str(item["label"])] += 1
|
| 93 |
+
count += 1
|
| 94 |
+
temporary.replace(destination)
|
| 95 |
+
result["splits"][split] = {"path": str(destination), "rows": count, "sha256": digest(destination), "label_counts": dict(sorted(labels.items())), "labelled": all(int(label) >= 0 for label in labels), "source_parquets": source_files}
|
| 96 |
+
print(f"{task} {split}: {count:,} rows; labels={dict(labels)}", flush=True)
|
| 97 |
+
return result
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def main() -> None:
|
| 101 |
+
ROOT.mkdir(parents=True, exist_ok=True)
|
| 102 |
+
pa.set_cpu_count(2)
|
| 103 |
+
pa.set_io_thread_count(2)
|
| 104 |
+
with ThreadPoolExecutor(max_workers=2) as pool:
|
| 105 |
+
prepared = list(pool.map(prepare_source, SOURCES))
|
| 106 |
+
manifest = {"prepared_at_utc": datetime.now(timezone.utc).isoformat(), "tasks": {spec[0]: result for spec, result in zip(SOURCES, prepared)},
|
| 107 |
+
"split_policy": {"sst2": "Train on official train; validation is the only public labelled holdout. Official test labels are hidden (-1).",
|
| 108 |
+
"race_middle": "Train on official middle/train; select using middle/validation; evaluate once on middle/test."},
|
| 109 |
+
"normalization": "Exact input strings retained; sentiment labels are 0=negative,1=positive; RACE A/B/C/D mapped to integer 0/1/2/3; no examples generated or reassigned."}
|
| 110 |
+
manifest_path = ROOT / "dataset_manifest.json"
|
| 111 |
+
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
| 112 |
+
print(f"Manifest: {manifest_path}", flush=True)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
if __name__ == "__main__":
|
| 116 |
+
main()
|
task_finetuning/requirements.txt
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Install the CUDA build of PyTorch appropriate for your GPU first.
|
| 2 |
+
torch>=2.0
|
| 3 |
+
numpy
|
| 4 |
+
tiktoken
|
| 5 |
+
pyarrow
|
| 6 |
+
tzdata; platform_system == "Windows"
|
| 7 |
+
# Optional: lower process priority while training.
|
| 8 |
+
psutil
|
task_finetuning/results/WiggleGPT_Generation_Record.json
ADDED
|
@@ -0,0 +1,1647 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"title": "WiggleGPT original-model generation record",
|
| 3 |
+
"capture_date": "26 October 2026",
|
| 4 |
+
"architecture_author": "P.C. O'Brien",
|
| 5 |
+
"document_collaborator": "Codex",
|
| 6 |
+
"showcase_selection": [
|
| 7 |
+
"instruction/creative",
|
| 8 |
+
"writing/old_library",
|
| 9 |
+
"instruction/summary",
|
| 10 |
+
"writing/sea_poem"
|
| 11 |
+
],
|
| 12 |
+
"selection_note": "Curated examples selected after reading the captures. This is a demonstration, not a random sample or an accuracy benchmark. One fixed-seed draw per prompt. Wording is unedited.",
|
| 13 |
+
"batches": {
|
| 14 |
+
"instruction": {
|
| 15 |
+
"date": "2026-10-26T05:19:01.936654+00:00",
|
| 16 |
+
"model": {
|
| 17 |
+
"checkpoint": "out-wigglegpt-finetune-native/ckpt.pt",
|
| 18 |
+
"saved_step": 9750,
|
| 19 |
+
"parameters": 123697920,
|
| 20 |
+
"model_args": {
|
| 21 |
+
"n_layer": 12,
|
| 22 |
+
"n_head": 12,
|
| 23 |
+
"n_embd": 768,
|
| 24 |
+
"block_size": 1024,
|
| 25 |
+
"bias": false,
|
| 26 |
+
"vocab_size": 50304,
|
| 27 |
+
"dropout": 0.0,
|
| 28 |
+
"use_bio_mlp": true,
|
| 29 |
+
"use_rmsnorm": true,
|
| 30 |
+
"use_rope": true,
|
| 31 |
+
"gradient_checkpointing": false
|
| 32 |
+
},
|
| 33 |
+
"torch": "2.11.0.dev20260215+cu128",
|
| 34 |
+
"checkpoint_loss_field": 1.5888433128595352,
|
| 35 |
+
"device": "NVIDIA GeForce RTX 5060 Ti",
|
| 36 |
+
"model_source": "model_bio.py",
|
| 37 |
+
"model_source_sha256": "a28201b8194941a75d3ca2e0812e6d2d0eef1b2656cc0c9f8fd0ca65bf96b28b",
|
| 38 |
+
"checkpoint_sha256": "d07ae2f80a7dcb5a3d662e66e1c526460e98f9d6d29803acaa7b015d592514d0"
|
| 39 |
+
},
|
| 40 |
+
"sampling": {
|
| 41 |
+
"max_new_tokens": 256,
|
| 42 |
+
"temperature": 0.7,
|
| 43 |
+
"top_k": 50,
|
| 44 |
+
"top_p": 1.0,
|
| 45 |
+
"repetition_penalty": 1.15,
|
| 46 |
+
"seed": 42
|
| 47 |
+
},
|
| 48 |
+
"precision": "bfloat16 autocast",
|
| 49 |
+
"system_prompt": null,
|
| 50 |
+
"retrieval": false,
|
| 51 |
+
"selection": "One draw per prompt; seed reset to 42 before each response.",
|
| 52 |
+
"sampler_source": "wiggle_assistant.py (direct sampler call only)",
|
| 53 |
+
"sampler_sha256": "fc10c18e3cd43986437c0f2ecde5a6b1c167423d9db98243e6f8c7e393711791",
|
| 54 |
+
"sampler_note": "Calls generate directly with original model_bio.py and original checkpoint. No Gosport adaptation weights, notes, Assistant wrapper or system prompt. Padded vocabulary IDs are masked; repetition penalty applies to generated tokens with sign-aware scaling; role terminators are removed.",
|
| 55 |
+
"results": [
|
| 56 |
+
{
|
| 57 |
+
"id": "explanation",
|
| 58 |
+
"prompt": "Explain why the sky is blue in simple terms.",
|
| 59 |
+
"serialised_prompt": "<|user|>\nExplain why the sky is blue in simple terms.\n<|/user|>\n<|assistant|>\n",
|
| 60 |
+
"answer": "The sky is a classic example of a reflection in the sky, where light from different sources intersects with one another. This phenomenon occurs because the wavelengths of light in each object are scattered across the same line of sight and appear to be reflecting off or fading away as they enter our eyes' domain.",
|
| 61 |
+
"seconds": 5.57,
|
| 62 |
+
"visible_text_tokens": 60,
|
| 63 |
+
"record_id": "instruction/explanation"
|
| 64 |
+
},
|
| 65 |
+
{
|
| 66 |
+
"id": "analogy",
|
| 67 |
+
"prompt": "Explain how a neural network learns, using a simple analogy.",
|
| 68 |
+
"serialised_prompt": "<|user|>\nExplain how a neural network learns, using a simple analogy.\n<|/user|>\n<|assistant|>\n",
|
| 69 |
+
"answer": "A neural network learns through a process called learning. In the context of the original analogy, we can think about it like this:\n```python\nimport numpy as np\nfrom scipy import scikit_image\n\n# Define the training data\ndata = {'Name': 'John', 'Age': 25, 'Score': 20}\nsensor = scikit-learn(data)\nprint(\"Sensor training example:\", sensor)\n```\nIn this code snippet, `scikit",
|
| 70 |
+
"seconds": 1.736,
|
| 71 |
+
"visible_text_tokens": 109,
|
| 72 |
+
"record_id": "instruction/analogy"
|
| 73 |
+
},
|
| 74 |
+
{
|
| 75 |
+
"id": "creative",
|
| 76 |
+
"prompt": "Write a short story about a lighthouse keeper who discovers a message in a bottle.",
|
| 77 |
+
"serialised_prompt": "<|user|>\nWrite a short story about a lighthouse keeper who discovers a message in a bottle.\n<|/user|>\n<|assistant|>\n",
|
| 78 |
+
"answer": "As the last remnants of sunlight faded, the lighthouse stood sentinel over the edge of the English countryside. The sounds of the sea and the creaking of old wooden beams stretched out before her like a whispered warning. The keeper, a young woman named Elara, had come to the aid of a shipwrecked sailor who had been stranded on the side of a dense forest for weeks. The message was etched in the glass: \"Lift your spirit. We have found you.\"",
|
| 79 |
+
"seconds": 1.662,
|
| 80 |
+
"visible_text_tokens": 97,
|
| 81 |
+
"record_id": "instruction/creative"
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"id": "dialogue",
|
| 85 |
+
"prompt": "Write a friendly conversation between a gardener and a curious robot.",
|
| 86 |
+
"serialised_prompt": "<|user|>\nWrite a friendly conversation between a gardener and a curious robot.\n<|/user|>\n<|assistant|>\n",
|
| 87 |
+
"answer": "The sun beats down on the rolling hills, casting a golden glow over the lush oases of this quaint village. As the sun sets, the gentle hum of garden plants and the soft rustle of leaves help to purify the air, making the villagers feel at peace despite the harsh weather conditions. The smell of blooming wildflowers wafts through the gardens, teasing the senses with its sweet fragrance that seems to match their own skin tones - a scent that draws the attention of passersby all day long.\"",
|
| 88 |
+
"seconds": 1.783,
|
| 89 |
+
"visible_text_tokens": 103,
|
| 90 |
+
"record_id": "instruction/dialogue"
|
| 91 |
+
},
|
| 92 |
+
{
|
| 93 |
+
"id": "summary",
|
| 94 |
+
"prompt": "Summarise this in one sentence: A community garden opened on Saturday. Twenty volunteers planted herbs and vegetables. The garden will provide free food for local families.",
|
| 95 |
+
"serialised_prompt": "<|user|>\nSummarise this in one sentence: A community garden opened on Saturday. Twenty volunteers planted herbs and vegetables. The garden will provide free food for local families.\n<|/user|>\n<|assistant|>\n",
|
| 96 |
+
"answer": "Here's a rewritten version of the sentence: The community garden opened on Saturday, providing free food for local families.",
|
| 97 |
+
"seconds": 0.464,
|
| 98 |
+
"visible_text_tokens": 23,
|
| 99 |
+
"record_id": "instruction/summary"
|
| 100 |
+
},
|
| 101 |
+
{
|
| 102 |
+
"id": "brainstorm",
|
| 103 |
+
"prompt": "Suggest three names for a friendly robot that helps people grow plants.",
|
| 104 |
+
"serialised_prompt": "<|user|>\nSuggest three names for a friendly robot that helps people grow plants.\n<|/user|>\n<|assistant|>\n",
|
| 105 |
+
"answer": "Here are three simple names for a friendly robot that helps people grow plants:\n```python\n# Define the name A, B and C for the garden\nA = [0] # 1.0 # 0.0b is acceptable (no need to specify any constants)\n\n def get_garden(self):\n # Create an empty space in the center where you can plant flowers - or not",
|
| 106 |
+
"seconds": 1.708,
|
| 107 |
+
"visible_text_tokens": 97,
|
| 108 |
+
"record_id": "instruction/brainstorm"
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"id": "practical",
|
| 112 |
+
"prompt": "How can I start growing herbs on a sunny kitchen windowsill?",
|
| 113 |
+
"serialised_prompt": "<|user|>\nHow can I start growing herbs on a sunny kitchen windowsill?\n<|/user|>\n<|assistant|>\n",
|
| 114 |
+
"answer": "Starting to grow herbs on a sunny kitchen windowill requires some creative thinking and planning. Here are some ideas to help you start:",
|
| 115 |
+
"seconds": 0.422,
|
| 116 |
+
"visible_text_tokens": 26,
|
| 117 |
+
"record_id": "instruction/practical"
|
| 118 |
+
},
|
| 119 |
+
{
|
| 120 |
+
"id": "science",
|
| 121 |
+
"prompt": "What is the difference between a planet and a star?",
|
| 122 |
+
"serialised_prompt": "<|user|>\nWhat is the difference between a planet and a star?\n<|/user|>\n<|assistant|>\n",
|
| 123 |
+
"answer": "A planet and a star are two distinct concepts that have different origins, shapes, and forms. Understanding the distinction between them is crucial for understanding the properties of planets in various planetary systems like stars or planets in orbit around their host star at the galactic center (GCC).",
|
| 124 |
+
"seconds": 1.063,
|
| 125 |
+
"visible_text_tokens": 54,
|
| 126 |
+
"record_id": "instruction/science"
|
| 127 |
+
},
|
| 128 |
+
{
|
| 129 |
+
"id": "arithmetic",
|
| 130 |
+
"prompt": "I have 3 boxes with 4 pencils in each box. How many pencils do I have altogether?",
|
| 131 |
+
"serialised_prompt": "<|user|>\nI have 3 boxes with 4 pencils in each box. How many pencils do I have altogether?\n<|/user|>\n<|assistant|>\n",
|
| 132 |
+
"answer": "You can use the same formula as before: 4 pencils in a box of three pencils. The formula is: (x - 2) * (3*1 + 1)(2)*(5-4), where x is the number of pencils, 2 is the number of pencils, and 3 is the total number of points on each box at that point along with the cost for the first two pencils or the remaining pencils if they are added together to form one larger square.",
|
| 133 |
+
"seconds": 1.751,
|
| 134 |
+
"visible_text_tokens": 101,
|
| 135 |
+
"record_id": "instruction/arithmetic"
|
| 136 |
+
},
|
| 137 |
+
{
|
| 138 |
+
"id": "code",
|
| 139 |
+
"prompt": "Write a Python function that returns the larger of two numbers.",
|
| 140 |
+
"serialised_prompt": "<|user|>\nWrite a Python function that returns the larger of two numbers.\n<|/user|>\n<|assistant|>\n",
|
| 141 |
+
"answer": "Here's a simple Python function to find the larger of two numbers:\n```python\ndef larger(nums):\n return nums[1] if nums[0] != nums[1] else \"b\"\n```\nThis function uses the built-in `sum` function from Python, which returns the sum of the first element (or more specifically, the second element) and the second element. The result is then returned as an array.",
|
| 142 |
+
"seconds": 1.666,
|
| 143 |
+
"visible_text_tokens": 100,
|
| 144 |
+
"record_id": "instruction/code"
|
| 145 |
+
}
|
| 146 |
+
]
|
| 147 |
+
},
|
| 148 |
+
"writing": {
|
| 149 |
+
"date": "2026-10-26T05:24:29.760095+00:00",
|
| 150 |
+
"model": {
|
| 151 |
+
"checkpoint": "out-wigglegpt-finetune-native/ckpt.pt",
|
| 152 |
+
"saved_step": 9750,
|
| 153 |
+
"parameters": 123697920,
|
| 154 |
+
"model_args": {
|
| 155 |
+
"n_layer": 12,
|
| 156 |
+
"n_head": 12,
|
| 157 |
+
"n_embd": 768,
|
| 158 |
+
"block_size": 1024,
|
| 159 |
+
"bias": false,
|
| 160 |
+
"vocab_size": 50304,
|
| 161 |
+
"dropout": 0.0,
|
| 162 |
+
"use_bio_mlp": true,
|
| 163 |
+
"use_rmsnorm": true,
|
| 164 |
+
"use_rope": true,
|
| 165 |
+
"gradient_checkpointing": false
|
| 166 |
+
},
|
| 167 |
+
"torch": "2.11.0.dev20260215+cu128",
|
| 168 |
+
"checkpoint_loss_field": 1.5888433128595352,
|
| 169 |
+
"device": "NVIDIA GeForce RTX 5060 Ti",
|
| 170 |
+
"model_source": "model_bio.py",
|
| 171 |
+
"model_source_sha256": "a28201b8194941a75d3ca2e0812e6d2d0eef1b2656cc0c9f8fd0ca65bf96b28b",
|
| 172 |
+
"checkpoint_sha256": "d07ae2f80a7dcb5a3d662e66e1c526460e98f9d6d29803acaa7b015d592514d0"
|
| 173 |
+
},
|
| 174 |
+
"sampling": {
|
| 175 |
+
"max_new_tokens": 256,
|
| 176 |
+
"temperature": 0.7,
|
| 177 |
+
"top_k": 50,
|
| 178 |
+
"top_p": 1.0,
|
| 179 |
+
"repetition_penalty": 1.15,
|
| 180 |
+
"seed": 42
|
| 181 |
+
},
|
| 182 |
+
"precision": "bfloat16 autocast",
|
| 183 |
+
"system_prompt": null,
|
| 184 |
+
"retrieval": false,
|
| 185 |
+
"selection": "One draw per prompt; seed reset to 42 before each response.",
|
| 186 |
+
"sampler_source": "wiggle_assistant.py (direct sampler call only)",
|
| 187 |
+
"sampler_sha256": "fc10c18e3cd43986437c0f2ecde5a6b1c167423d9db98243e6f8c7e393711791",
|
| 188 |
+
"sampler_note": "Calls generate directly with original model_bio.py and original checkpoint. No Gosport adaptation weights, notes, Assistant wrapper or system prompt. Padded vocabulary IDs are masked; repetition penalty applies to generated tokens with sign-aware scaling; role terminators are removed.",
|
| 189 |
+
"results": [
|
| 190 |
+
{
|
| 191 |
+
"id": "sunrise",
|
| 192 |
+
"prompt": "Write a short paragraph describing a peaceful garden at sunrise.",
|
| 193 |
+
"serialised_prompt": "<|user|>\nWrite a short paragraph describing a peaceful garden at sunrise.\n<|/user|>\n<|assistant|>\n",
|
| 194 |
+
"answer": "As the gentle breeze rustled the leaves of a small, intricately carved wooden box in the center of the garden's design, the soft chirping of crickets and gentle breezes filled the air. It was as if spring had been brought to life by the subtle stirrings of tiny flowers that flitted about beneath the surface like tiny orbs. But at the same time – there were still moments when it seemed that something was missing from this peaceful retreat; one moment, an unusual light burst through the canopy above, illuminating a nearby patch of lush greenery with its own unique properties.\"",
|
| 195 |
+
"seconds": 5.923,
|
| 196 |
+
"visible_text_tokens": 119,
|
| 197 |
+
"record_id": "writing/sunrise"
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"id": "robot_story",
|
| 201 |
+
"prompt": "Write a short story about a robot who learns to paint.",
|
| 202 |
+
"serialised_prompt": "<|user|>\nWrite a short story about a robot who learns to paint.\n<|/user|>\n<|assistant|>\n",
|
| 203 |
+
"answer": "As the last rays of sunlight faded, Jack had taken a deep breath and reached out to touch his brush. The world around her was a blur as she worked on a small piece of paper again - a single sentence from her favorite author, a quilt he had made for her at home in the city. The words were like a whispered secret: she was a painter, but it wasn't just about creating something beautiful; painting was a skill that could bring joy wherever death seemed to be a part of life.\"",
|
| 204 |
+
"seconds": 1.705,
|
| 205 |
+
"visible_text_tokens": 103,
|
| 206 |
+
"record_id": "writing/robot_story"
|
| 207 |
+
},
|
| 208 |
+
{
|
| 209 |
+
"id": "sea_poem",
|
| 210 |
+
"prompt": "Write a poem about the sea.",
|
| 211 |
+
"serialised_prompt": "<|user|>\nWrite a poem about the sea.\n<|/user|>\n<|assistant|>\n",
|
| 212 |
+
"answer": "The sea's secrets,\nIts depths are a treasure trove o' gold.",
|
| 213 |
+
"seconds": 0.383,
|
| 214 |
+
"visible_text_tokens": 16,
|
| 215 |
+
"record_id": "writing/sea_poem"
|
| 216 |
+
},
|
| 217 |
+
{
|
| 218 |
+
"id": "old_library",
|
| 219 |
+
"prompt": "Describe an old library on a rainy afternoon.",
|
| 220 |
+
"serialised_prompt": "<|user|>\nDescribe an old library on a rainy afternoon.\n<|/user|>\n<|assistant|>\n",
|
| 221 |
+
"answer": "The old library that's been here for decades is a treasure trove of memories. As I gaze out the window, it's as if time has taken its toll on this small town like ours does. The air is thick with dust and cobwebs, and the only sounds are the creaking of wooden floorboards beneath my feet and the occasional distant hum of a nearby train.",
|
| 222 |
+
"seconds": 1.163,
|
| 223 |
+
"visible_text_tokens": 76,
|
| 224 |
+
"record_id": "writing/old_library"
|
| 225 |
+
},
|
| 226 |
+
{
|
| 227 |
+
"id": "rewrite",
|
| 228 |
+
"prompt": "Rewrite this sentence to sound more cheerful: I am going for a walk.",
|
| 229 |
+
"serialised_prompt": "<|user|>\nRewrite this sentence to sound more cheerful: I am going for a walk.\n<|/user|>\n<|assistant|>\n",
|
| 230 |
+
"answer": "Here's a rewritten version of the sentence: I am going for a walk.",
|
| 231 |
+
"seconds": 0.367,
|
| 232 |
+
"visible_text_tokens": 16,
|
| 233 |
+
"record_id": "writing/rewrite"
|
| 234 |
+
},
|
| 235 |
+
{
|
| 236 |
+
"id": "thanks",
|
| 237 |
+
"prompt": "Write a thank-you message to volunteers who planted a community garden.",
|
| 238 |
+
"serialised_prompt": "<|user|>\nWrite a thank-you message to volunteers who planted a community garden.\n<|/user|>\n<|assistant|>\n",
|
| 239 |
+
"answer": "Here's a thank-you message to volunteers who planted the community garden:\n```python\n# Begin with a clear greeting and explain your programming language.\n\n class Program\n * @param user_name _\n * @returns \"Welcome, my name is John Doe.\"",
|
| 240 |
+
"seconds": 1.012,
|
| 241 |
+
"visible_text_tokens": 64,
|
| 242 |
+
"record_id": "writing/thanks"
|
| 243 |
+
}
|
| 244 |
+
]
|
| 245 |
+
},
|
| 246 |
+
"pretrained": {
|
| 247 |
+
"date": "2026-10-26T05:21:46.877200+00:00",
|
| 248 |
+
"model": {
|
| 249 |
+
"checkpoint": "out-wigglegpt-pure-124m/ckpt.pt",
|
| 250 |
+
"saved_step": 600000,
|
| 251 |
+
"parameters": 123697920,
|
| 252 |
+
"model_args": {
|
| 253 |
+
"n_layer": 12,
|
| 254 |
+
"n_head": 12,
|
| 255 |
+
"n_embd": 768,
|
| 256 |
+
"block_size": 1024,
|
| 257 |
+
"bias": false,
|
| 258 |
+
"vocab_size": 50304,
|
| 259 |
+
"dropout": 0.0,
|
| 260 |
+
"use_bio_mlp": true,
|
| 261 |
+
"use_rmsnorm": true,
|
| 262 |
+
"use_rope": true,
|
| 263 |
+
"gradient_checkpointing": false
|
| 264 |
+
},
|
| 265 |
+
"torch": "2.11.0.dev20260215+cu128",
|
| 266 |
+
"checkpoint_loss_field": 3.1621146202087402,
|
| 267 |
+
"device": "NVIDIA GeForce RTX 5060 Ti",
|
| 268 |
+
"checkpoint_sha256": "88afaab6c93ef3e33940b4e25cbce2f44038c428109600afe8285586a3a767b9"
|
| 269 |
+
},
|
| 270 |
+
"sampling": {
|
| 271 |
+
"max_new_tokens": 160,
|
| 272 |
+
"temperature": 0.7,
|
| 273 |
+
"top_k": 50,
|
| 274 |
+
"top_p": 1.0,
|
| 275 |
+
"repetition_penalty": 1.15,
|
| 276 |
+
"seed": 42
|
| 277 |
+
},
|
| 278 |
+
"precision": "bfloat16 autocast",
|
| 279 |
+
"system_prompt": null,
|
| 280 |
+
"retrieval": false,
|
| 281 |
+
"input_mode": "Plain text prefix, without conversation tags.",
|
| 282 |
+
"selection": "One draw per prompt; seed reset to 42 before each response.",
|
| 283 |
+
"results": [
|
| 284 |
+
{
|
| 285 |
+
"id": "harbour",
|
| 286 |
+
"prompt": "On a quiet evening by the harbour, a small robot sat beside the lighthouse and watched",
|
| 287 |
+
"answer": " for the signal that it was time to go.\n\nThe robot, called M1, was a small industrial drone (a type of mobile phone for drones) weighing around 16 kilograms each with a maximum weight of 300 kilograms and carrying an antennae capable about 100 metres long using two hand sensors mounted on top along its mouth like a drone’s nose at 20 degrees angle from the ground in the bay.\n\nIt had no purpose but almost everything else was going well: it was a low-tech, portable home automation platform for drones operating underwater, and there were more than 50 million registered drones registered worldwide today – enough for a year total of 1 billion.\n\nThis was a big deal because, as you may know by now if I had been asked how many times have I",
|
| 288 |
+
"token_ids": [
|
| 289 |
+
329,
|
| 290 |
+
262,
|
| 291 |
+
6737,
|
| 292 |
+
326,
|
| 293 |
+
340,
|
| 294 |
+
373,
|
| 295 |
+
640,
|
| 296 |
+
284,
|
| 297 |
+
467,
|
| 298 |
+
13,
|
| 299 |
+
198,
|
| 300 |
+
198,
|
| 301 |
+
464,
|
| 302 |
+
9379,
|
| 303 |
+
11,
|
| 304 |
+
1444,
|
| 305 |
+
337,
|
| 306 |
+
16,
|
| 307 |
+
11,
|
| 308 |
+
373,
|
| 309 |
+
257,
|
| 310 |
+
1402,
|
| 311 |
+
7593,
|
| 312 |
+
12170,
|
| 313 |
+
357,
|
| 314 |
+
64,
|
| 315 |
+
2099,
|
| 316 |
+
286,
|
| 317 |
+
5175,
|
| 318 |
+
3072,
|
| 319 |
+
329,
|
| 320 |
+
15382,
|
| 321 |
+
8,
|
| 322 |
+
21990,
|
| 323 |
+
1088,
|
| 324 |
+
1467,
|
| 325 |
+
37075,
|
| 326 |
+
1123,
|
| 327 |
+
351,
|
| 328 |
+
257,
|
| 329 |
+
5415,
|
| 330 |
+
3463,
|
| 331 |
+
286,
|
| 332 |
+
5867,
|
| 333 |
+
37075,
|
| 334 |
+
290,
|
| 335 |
+
6872,
|
| 336 |
+
281,
|
| 337 |
+
20509,
|
| 338 |
+
68,
|
| 339 |
+
6007,
|
| 340 |
+
546,
|
| 341 |
+
1802,
|
| 342 |
+
18985,
|
| 343 |
+
890,
|
| 344 |
+
1262,
|
| 345 |
+
734,
|
| 346 |
+
1021,
|
| 347 |
+
15736,
|
| 348 |
+
12623,
|
| 349 |
+
319,
|
| 350 |
+
1353,
|
| 351 |
+
1863,
|
| 352 |
+
663,
|
| 353 |
+
5422,
|
| 354 |
+
588,
|
| 355 |
+
257,
|
| 356 |
+
12170,
|
| 357 |
+
447,
|
| 358 |
+
247,
|
| 359 |
+
82,
|
| 360 |
+
9686,
|
| 361 |
+
379,
|
| 362 |
+
1160,
|
| 363 |
+
7370,
|
| 364 |
+
9848,
|
| 365 |
+
422,
|
| 366 |
+
262,
|
| 367 |
+
2323,
|
| 368 |
+
287,
|
| 369 |
+
262,
|
| 370 |
+
15489,
|
| 371 |
+
13,
|
| 372 |
+
198,
|
| 373 |
+
198,
|
| 374 |
+
1026,
|
| 375 |
+
550,
|
| 376 |
+
645,
|
| 377 |
+
4007,
|
| 378 |
+
475,
|
| 379 |
+
2048,
|
| 380 |
+
2279,
|
| 381 |
+
2073,
|
| 382 |
+
373,
|
| 383 |
+
1016,
|
| 384 |
+
880,
|
| 385 |
+
25,
|
| 386 |
+
340,
|
| 387 |
+
373,
|
| 388 |
+
257,
|
| 389 |
+
1877,
|
| 390 |
+
12,
|
| 391 |
+
13670,
|
| 392 |
+
11,
|
| 393 |
+
17726,
|
| 394 |
+
1363,
|
| 395 |
+
22771,
|
| 396 |
+
3859,
|
| 397 |
+
329,
|
| 398 |
+
15382,
|
| 399 |
+
5361,
|
| 400 |
+
21258,
|
| 401 |
+
11,
|
| 402 |
+
290,
|
| 403 |
+
612,
|
| 404 |
+
547,
|
| 405 |
+
517,
|
| 406 |
+
621,
|
| 407 |
+
2026,
|
| 408 |
+
1510,
|
| 409 |
+
6823,
|
| 410 |
+
15382,
|
| 411 |
+
6823,
|
| 412 |
+
8688,
|
| 413 |
+
1909,
|
| 414 |
+
784,
|
| 415 |
+
1576,
|
| 416 |
+
329,
|
| 417 |
+
257,
|
| 418 |
+
614,
|
| 419 |
+
2472,
|
| 420 |
+
286,
|
| 421 |
+
352,
|
| 422 |
+
2997,
|
| 423 |
+
13,
|
| 424 |
+
198,
|
| 425 |
+
198,
|
| 426 |
+
1212,
|
| 427 |
+
373,
|
| 428 |
+
257,
|
| 429 |
+
1263,
|
| 430 |
+
1730,
|
| 431 |
+
780,
|
| 432 |
+
11,
|
| 433 |
+
355,
|
| 434 |
+
345,
|
| 435 |
+
743,
|
| 436 |
+
760,
|
| 437 |
+
416,
|
| 438 |
+
783,
|
| 439 |
+
611,
|
| 440 |
+
314,
|
| 441 |
+
550,
|
| 442 |
+
587,
|
| 443 |
+
1965,
|
| 444 |
+
703,
|
| 445 |
+
867,
|
| 446 |
+
1661,
|
| 447 |
+
423,
|
| 448 |
+
314
|
| 449 |
+
],
|
| 450 |
+
"seconds": 7.619,
|
| 451 |
+
"stop_reason": "token_limit",
|
| 452 |
+
"record_id": "pretrained/harbour"
|
| 453 |
+
},
|
| 454 |
+
{
|
| 455 |
+
"id": "mystery",
|
| 456 |
+
"prompt": "At the bottom of the old wooden chest, Maya found a brass key and a note that read:",
|
| 457 |
+
"answer": " \"She's a slave.\"\n\nManga artist and composer, Robert M. Koehler, was at the time obsessed with making the most of Maya's wealth (he even had his own collection of art from the old age) so he created a series about her life as slave girl — and sometimes as an independent woman who worked hard to earn it all back up:\n\nI saw the world through Maya's eyes in every detail but she wasn't allowed - or not allowed by all. Maya is a young, beautiful woman that I have only now realized could be the real face behind the very best and best of Maya. Every day she goes by without looking for anything... I didn’t know how you did this! I wonder if she was going crazy yet.. maybe I",
|
| 458 |
+
"token_ids": [
|
| 459 |
+
366,
|
| 460 |
+
3347,
|
| 461 |
+
338,
|
| 462 |
+
257,
|
| 463 |
+
11778,
|
| 464 |
+
526,
|
| 465 |
+
198,
|
| 466 |
+
198,
|
| 467 |
+
44,
|
| 468 |
+
16484,
|
| 469 |
+
6802,
|
| 470 |
+
290,
|
| 471 |
+
26777,
|
| 472 |
+
11,
|
| 473 |
+
5199,
|
| 474 |
+
337,
|
| 475 |
+
13,
|
| 476 |
+
509,
|
| 477 |
+
2577,
|
| 478 |
+
49737,
|
| 479 |
+
11,
|
| 480 |
+
373,
|
| 481 |
+
379,
|
| 482 |
+
262,
|
| 483 |
+
640,
|
| 484 |
+
21366,
|
| 485 |
+
351,
|
| 486 |
+
1642,
|
| 487 |
+
262,
|
| 488 |
+
749,
|
| 489 |
+
286,
|
| 490 |
+
26041,
|
| 491 |
+
338,
|
| 492 |
+
5129,
|
| 493 |
+
357,
|
| 494 |
+
258,
|
| 495 |
+
772,
|
| 496 |
+
550,
|
| 497 |
+
465,
|
| 498 |
+
898,
|
| 499 |
+
4947,
|
| 500 |
+
286,
|
| 501 |
+
1242,
|
| 502 |
+
422,
|
| 503 |
+
262,
|
| 504 |
+
1468,
|
| 505 |
+
2479,
|
| 506 |
+
8,
|
| 507 |
+
523,
|
| 508 |
+
339,
|
| 509 |
+
2727,
|
| 510 |
+
257,
|
| 511 |
+
2168,
|
| 512 |
+
546,
|
| 513 |
+
607,
|
| 514 |
+
1204,
|
| 515 |
+
355,
|
| 516 |
+
11778,
|
| 517 |
+
2576,
|
| 518 |
+
851,
|
| 519 |
+
290,
|
| 520 |
+
3360,
|
| 521 |
+
355,
|
| 522 |
+
281,
|
| 523 |
+
4795,
|
| 524 |
+
2415,
|
| 525 |
+
508,
|
| 526 |
+
3111,
|
| 527 |
+
1327,
|
| 528 |
+
284,
|
| 529 |
+
5160,
|
| 530 |
+
340,
|
| 531 |
+
477,
|
| 532 |
+
736,
|
| 533 |
+
510,
|
| 534 |
+
25,
|
| 535 |
+
198,
|
| 536 |
+
198,
|
| 537 |
+
40,
|
| 538 |
+
2497,
|
| 539 |
+
262,
|
| 540 |
+
995,
|
| 541 |
+
832,
|
| 542 |
+
26041,
|
| 543 |
+
338,
|
| 544 |
+
2951,
|
| 545 |
+
287,
|
| 546 |
+
790,
|
| 547 |
+
3703,
|
| 548 |
+
475,
|
| 549 |
+
673,
|
| 550 |
+
2492,
|
| 551 |
+
470,
|
| 552 |
+
3142,
|
| 553 |
+
532,
|
| 554 |
+
393,
|
| 555 |
+
407,
|
| 556 |
+
3142,
|
| 557 |
+
416,
|
| 558 |
+
477,
|
| 559 |
+
13,
|
| 560 |
+
26041,
|
| 561 |
+
318,
|
| 562 |
+
257,
|
| 563 |
+
1862,
|
| 564 |
+
11,
|
| 565 |
+
4950,
|
| 566 |
+
2415,
|
| 567 |
+
326,
|
| 568 |
+
314,
|
| 569 |
+
423,
|
| 570 |
+
691,
|
| 571 |
+
783,
|
| 572 |
+
6939,
|
| 573 |
+
714,
|
| 574 |
+
307,
|
| 575 |
+
262,
|
| 576 |
+
1103,
|
| 577 |
+
1986,
|
| 578 |
+
2157,
|
| 579 |
+
262,
|
| 580 |
+
845,
|
| 581 |
+
1266,
|
| 582 |
+
290,
|
| 583 |
+
1266,
|
| 584 |
+
286,
|
| 585 |
+
26041,
|
| 586 |
+
13,
|
| 587 |
+
3887,
|
| 588 |
+
1110,
|
| 589 |
+
673,
|
| 590 |
+
2925,
|
| 591 |
+
416,
|
| 592 |
+
1231,
|
| 593 |
+
2045,
|
| 594 |
+
329,
|
| 595 |
+
1997,
|
| 596 |
+
986,
|
| 597 |
+
314,
|
| 598 |
+
1422,
|
| 599 |
+
447,
|
| 600 |
+
247,
|
| 601 |
+
83,
|
| 602 |
+
760,
|
| 603 |
+
703,
|
| 604 |
+
345,
|
| 605 |
+
750,
|
| 606 |
+
428,
|
| 607 |
+
0,
|
| 608 |
+
314,
|
| 609 |
+
4240,
|
| 610 |
+
611,
|
| 611 |
+
673,
|
| 612 |
+
373,
|
| 613 |
+
1016,
|
| 614 |
+
7165,
|
| 615 |
+
1865,
|
| 616 |
+
492,
|
| 617 |
+
3863,
|
| 618 |
+
314
|
| 619 |
+
],
|
| 620 |
+
"seconds": 2.64,
|
| 621 |
+
"stop_reason": "token_limit",
|
| 622 |
+
"record_id": "pretrained/mystery"
|
| 623 |
+
},
|
| 624 |
+
{
|
| 625 |
+
"id": "garden",
|
| 626 |
+
"prompt": "Urban community gardens can bring people together in several ways. First,",
|
| 627 |
+
"answer": " they can be simple things like a flower garden or a kitchen, and then it’s just a space to make some sort of connection.\n\nSecond is the variety in how you use your food. It may seem strange at first — but if we look into the world now more closely than ever before (the food economy has grown significantly), we find that we have lost our way with many typesof foods: “all natural”; ‘natural-looking' – especially those that are made from nature; ‘good plant-based' (in my opinion) as well such as corn and wheat; and so on….\n\nWhen I was growing up here all over the city there were only a few types of produce out…I love the fact people could actually buy",
|
| 628 |
+
"token_ids": [
|
| 629 |
+
484,
|
| 630 |
+
460,
|
| 631 |
+
307,
|
| 632 |
+
2829,
|
| 633 |
+
1243,
|
| 634 |
+
588,
|
| 635 |
+
257,
|
| 636 |
+
15061,
|
| 637 |
+
11376,
|
| 638 |
+
393,
|
| 639 |
+
257,
|
| 640 |
+
9592,
|
| 641 |
+
11,
|
| 642 |
+
290,
|
| 643 |
+
788,
|
| 644 |
+
340,
|
| 645 |
+
447,
|
| 646 |
+
247,
|
| 647 |
+
82,
|
| 648 |
+
655,
|
| 649 |
+
257,
|
| 650 |
+
2272,
|
| 651 |
+
284,
|
| 652 |
+
787,
|
| 653 |
+
617,
|
| 654 |
+
3297,
|
| 655 |
+
286,
|
| 656 |
+
4637,
|
| 657 |
+
13,
|
| 658 |
+
198,
|
| 659 |
+
198,
|
| 660 |
+
12211,
|
| 661 |
+
318,
|
| 662 |
+
262,
|
| 663 |
+
4996,
|
| 664 |
+
287,
|
| 665 |
+
703,
|
| 666 |
+
345,
|
| 667 |
+
779,
|
| 668 |
+
534,
|
| 669 |
+
2057,
|
| 670 |
+
13,
|
| 671 |
+
632,
|
| 672 |
+
743,
|
| 673 |
+
1283,
|
| 674 |
+
6283,
|
| 675 |
+
379,
|
| 676 |
+
717,
|
| 677 |
+
851,
|
| 678 |
+
475,
|
| 679 |
+
611,
|
| 680 |
+
356,
|
| 681 |
+
804,
|
| 682 |
+
656,
|
| 683 |
+
262,
|
| 684 |
+
995,
|
| 685 |
+
783,
|
| 686 |
+
517,
|
| 687 |
+
7173,
|
| 688 |
+
621,
|
| 689 |
+
1683,
|
| 690 |
+
878,
|
| 691 |
+
357,
|
| 692 |
+
1169,
|
| 693 |
+
2057,
|
| 694 |
+
3773,
|
| 695 |
+
468,
|
| 696 |
+
7334,
|
| 697 |
+
5566,
|
| 698 |
+
828,
|
| 699 |
+
356,
|
| 700 |
+
1064,
|
| 701 |
+
326,
|
| 702 |
+
356,
|
| 703 |
+
423,
|
| 704 |
+
2626,
|
| 705 |
+
674,
|
| 706 |
+
835,
|
| 707 |
+
351,
|
| 708 |
+
867,
|
| 709 |
+
3858,
|
| 710 |
+
1659,
|
| 711 |
+
9013,
|
| 712 |
+
25,
|
| 713 |
+
564,
|
| 714 |
+
250,
|
| 715 |
+
439,
|
| 716 |
+
3288,
|
| 717 |
+
447,
|
| 718 |
+
251,
|
| 719 |
+
26,
|
| 720 |
+
564,
|
| 721 |
+
246,
|
| 722 |
+
11802,
|
| 723 |
+
12,
|
| 724 |
+
11534,
|
| 725 |
+
6,
|
| 726 |
+
784,
|
| 727 |
+
2592,
|
| 728 |
+
883,
|
| 729 |
+
326,
|
| 730 |
+
389,
|
| 731 |
+
925,
|
| 732 |
+
422,
|
| 733 |
+
3450,
|
| 734 |
+
26,
|
| 735 |
+
564,
|
| 736 |
+
246,
|
| 737 |
+
11274,
|
| 738 |
+
4618,
|
| 739 |
+
12,
|
| 740 |
+
3106,
|
| 741 |
+
6,
|
| 742 |
+
357,
|
| 743 |
+
259,
|
| 744 |
+
616,
|
| 745 |
+
4459,
|
| 746 |
+
8,
|
| 747 |
+
355,
|
| 748 |
+
880,
|
| 749 |
+
884,
|
| 750 |
+
355,
|
| 751 |
+
11676,
|
| 752 |
+
290,
|
| 753 |
+
17135,
|
| 754 |
+
26,
|
| 755 |
+
290,
|
| 756 |
+
523,
|
| 757 |
+
319,
|
| 758 |
+
11580,
|
| 759 |
+
198,
|
| 760 |
+
198,
|
| 761 |
+
2215,
|
| 762 |
+
314,
|
| 763 |
+
373,
|
| 764 |
+
3957,
|
| 765 |
+
510,
|
| 766 |
+
994,
|
| 767 |
+
477,
|
| 768 |
+
625,
|
| 769 |
+
262,
|
| 770 |
+
1748,
|
| 771 |
+
612,
|
| 772 |
+
547,
|
| 773 |
+
691,
|
| 774 |
+
257,
|
| 775 |
+
1178,
|
| 776 |
+
3858,
|
| 777 |
+
286,
|
| 778 |
+
4439,
|
| 779 |
+
503,
|
| 780 |
+
1399,
|
| 781 |
+
40,
|
| 782 |
+
1842,
|
| 783 |
+
262,
|
| 784 |
+
1109,
|
| 785 |
+
661,
|
| 786 |
+
714,
|
| 787 |
+
1682,
|
| 788 |
+
2822
|
| 789 |
+
],
|
| 790 |
+
"seconds": 2.608,
|
| 791 |
+
"stop_reason": "token_limit",
|
| 792 |
+
"record_id": "pretrained/garden"
|
| 793 |
+
},
|
| 794 |
+
{
|
| 795 |
+
"id": "ai",
|
| 796 |
+
"prompt": "Artificial intelligence is a field of computer science concerned with building systems that",
|
| 797 |
+
"answer": " can learn and adapt to new environments, such as natural resources in nature. The work has been performed with the help of the AIPT project and the collaboration between the University of Georgia Tech School for Engineering Science (USGS), the University of California, and the International Center for Systems Research at the University College London.\n\nThe researchers have also built a system that learns from different types of data over time – including information about the environment around it; how well food is delivered across distances by a person’s actions on an island or street level rather than through a tree stump flooring operationally based upon where you walk.\n\nIn each case they have developed methods used within a system called “nano-learning systems: the ultimate goal are learning the principles underlying artificial intelligence algorithms",
|
| 798 |
+
"token_ids": [
|
| 799 |
+
460,
|
| 800 |
+
2193,
|
| 801 |
+
290,
|
| 802 |
+
6068,
|
| 803 |
+
284,
|
| 804 |
+
649,
|
| 805 |
+
12493,
|
| 806 |
+
11,
|
| 807 |
+
884,
|
| 808 |
+
355,
|
| 809 |
+
3288,
|
| 810 |
+
4133,
|
| 811 |
+
287,
|
| 812 |
+
3450,
|
| 813 |
+
13,
|
| 814 |
+
383,
|
| 815 |
+
670,
|
| 816 |
+
468,
|
| 817 |
+
587,
|
| 818 |
+
6157,
|
| 819 |
+
351,
|
| 820 |
+
262,
|
| 821 |
+
1037,
|
| 822 |
+
286,
|
| 823 |
+
262,
|
| 824 |
+
317,
|
| 825 |
+
4061,
|
| 826 |
+
51,
|
| 827 |
+
1628,
|
| 828 |
+
290,
|
| 829 |
+
262,
|
| 830 |
+
12438,
|
| 831 |
+
1022,
|
| 832 |
+
262,
|
| 833 |
+
2059,
|
| 834 |
+
286,
|
| 835 |
+
7859,
|
| 836 |
+
9634,
|
| 837 |
+
3961,
|
| 838 |
+
329,
|
| 839 |
+
14044,
|
| 840 |
+
5800,
|
| 841 |
+
357,
|
| 842 |
+
2937,
|
| 843 |
+
14313,
|
| 844 |
+
828,
|
| 845 |
+
262,
|
| 846 |
+
2059,
|
| 847 |
+
286,
|
| 848 |
+
3442,
|
| 849 |
+
11,
|
| 850 |
+
290,
|
| 851 |
+
262,
|
| 852 |
+
4037,
|
| 853 |
+
3337,
|
| 854 |
+
329,
|
| 855 |
+
11998,
|
| 856 |
+
4992,
|
| 857 |
+
379,
|
| 858 |
+
262,
|
| 859 |
+
2059,
|
| 860 |
+
5535,
|
| 861 |
+
3576,
|
| 862 |
+
13,
|
| 863 |
+
198,
|
| 864 |
+
198,
|
| 865 |
+
464,
|
| 866 |
+
4837,
|
| 867 |
+
423,
|
| 868 |
+
635,
|
| 869 |
+
3170,
|
| 870 |
+
257,
|
| 871 |
+
1080,
|
| 872 |
+
326,
|
| 873 |
+
22974,
|
| 874 |
+
422,
|
| 875 |
+
1180,
|
| 876 |
+
3858,
|
| 877 |
+
286,
|
| 878 |
+
1366,
|
| 879 |
+
625,
|
| 880 |
+
640,
|
| 881 |
+
784,
|
| 882 |
+
1390,
|
| 883 |
+
1321,
|
| 884 |
+
546,
|
| 885 |
+
262,
|
| 886 |
+
2858,
|
| 887 |
+
1088,
|
| 888 |
+
340,
|
| 889 |
+
26,
|
| 890 |
+
703,
|
| 891 |
+
880,
|
| 892 |
+
2057,
|
| 893 |
+
318,
|
| 894 |
+
6793,
|
| 895 |
+
1973,
|
| 896 |
+
18868,
|
| 897 |
+
416,
|
| 898 |
+
257,
|
| 899 |
+
1048,
|
| 900 |
+
447,
|
| 901 |
+
247,
|
| 902 |
+
82,
|
| 903 |
+
4028,
|
| 904 |
+
319,
|
| 905 |
+
281,
|
| 906 |
+
7022,
|
| 907 |
+
393,
|
| 908 |
+
4675,
|
| 909 |
+
1241,
|
| 910 |
+
2138,
|
| 911 |
+
621,
|
| 912 |
+
832,
|
| 913 |
+
257,
|
| 914 |
+
5509,
|
| 915 |
+
42497,
|
| 916 |
+
4314,
|
| 917 |
+
278,
|
| 918 |
+
4905,
|
| 919 |
+
453,
|
| 920 |
+
1912,
|
| 921 |
+
2402,
|
| 922 |
+
810,
|
| 923 |
+
345,
|
| 924 |
+
2513,
|
| 925 |
+
13,
|
| 926 |
+
198,
|
| 927 |
+
198,
|
| 928 |
+
818,
|
| 929 |
+
1123,
|
| 930 |
+
1339,
|
| 931 |
+
484,
|
| 932 |
+
423,
|
| 933 |
+
4166,
|
| 934 |
+
5050,
|
| 935 |
+
973,
|
| 936 |
+
1626,
|
| 937 |
+
257,
|
| 938 |
+
1080,
|
| 939 |
+
1444,
|
| 940 |
+
564,
|
| 941 |
+
250,
|
| 942 |
+
77,
|
| 943 |
+
5733,
|
| 944 |
+
12,
|
| 945 |
+
40684,
|
| 946 |
+
3341,
|
| 947 |
+
25,
|
| 948 |
+
262,
|
| 949 |
+
8713,
|
| 950 |
+
3061,
|
| 951 |
+
389,
|
| 952 |
+
4673,
|
| 953 |
+
262,
|
| 954 |
+
7811,
|
| 955 |
+
10238,
|
| 956 |
+
11666,
|
| 957 |
+
4430,
|
| 958 |
+
16113
|
| 959 |
+
],
|
| 960 |
+
"seconds": 7.718,
|
| 961 |
+
"stop_reason": "token_limit",
|
| 962 |
+
"record_id": "pretrained/ai"
|
| 963 |
+
},
|
| 964 |
+
{
|
| 965 |
+
"id": "recipe",
|
| 966 |
+
"prompt": "Recipe: Simple tomato soup\nIngredients: tomatoes, onion, garlic, vegetable stock.\nMethod:\n1.",
|
| 967 |
+
"answer": " Combine the tomatoes and onion in a large bowl.\n\n2, except for the garlic, let it sit at room temperature until ready to cook over medium heat (about 3 minutes).\n\n3., stir together the garlic and onions into a small saucepan with about 1 cup of water and 1/4 tsp. of basil.\n\n4.) Add a pinch or two of the tomato paste to the rice. I like to use a fork if my thumb is on the wrong side as that will cause my skin to break down quickly after adding more water than necessary!\n\n5.). After cooking all your tomato paste ingredients you should be able to get rid off this time by simply filling up the bottom layer with some more vegetable stock while the remaining mixture remains soft so they can’t",
|
| 968 |
+
"token_ids": [
|
| 969 |
+
29176,
|
| 970 |
+
262,
|
| 971 |
+
23972,
|
| 972 |
+
290,
|
| 973 |
+
21670,
|
| 974 |
+
287,
|
| 975 |
+
257,
|
| 976 |
+
1588,
|
| 977 |
+
9396,
|
| 978 |
+
13,
|
| 979 |
+
198,
|
| 980 |
+
198,
|
| 981 |
+
17,
|
| 982 |
+
11,
|
| 983 |
+
2845,
|
| 984 |
+
329,
|
| 985 |
+
262,
|
| 986 |
+
17644,
|
| 987 |
+
11,
|
| 988 |
+
1309,
|
| 989 |
+
340,
|
| 990 |
+
1650,
|
| 991 |
+
379,
|
| 992 |
+
2119,
|
| 993 |
+
5951,
|
| 994 |
+
1566,
|
| 995 |
+
3492,
|
| 996 |
+
284,
|
| 997 |
+
4255,
|
| 998 |
+
625,
|
| 999 |
+
7090,
|
| 1000 |
+
4894,
|
| 1001 |
+
357,
|
| 1002 |
+
10755,
|
| 1003 |
+
513,
|
| 1004 |
+
2431,
|
| 1005 |
+
737,
|
| 1006 |
+
198,
|
| 1007 |
+
198,
|
| 1008 |
+
18,
|
| 1009 |
+
1539,
|
| 1010 |
+
11240,
|
| 1011 |
+
1978,
|
| 1012 |
+
262,
|
| 1013 |
+
17644,
|
| 1014 |
+
290,
|
| 1015 |
+
24681,
|
| 1016 |
+
656,
|
| 1017 |
+
257,
|
| 1018 |
+
1402,
|
| 1019 |
+
10746,
|
| 1020 |
+
6839,
|
| 1021 |
+
351,
|
| 1022 |
+
546,
|
| 1023 |
+
352,
|
| 1024 |
+
6508,
|
| 1025 |
+
286,
|
| 1026 |
+
1660,
|
| 1027 |
+
290,
|
| 1028 |
+
352,
|
| 1029 |
+
14,
|
| 1030 |
+
19,
|
| 1031 |
+
23053,
|
| 1032 |
+
13,
|
| 1033 |
+
286,
|
| 1034 |
+
37792,
|
| 1035 |
+
13,
|
| 1036 |
+
198,
|
| 1037 |
+
198,
|
| 1038 |
+
19,
|
| 1039 |
+
2014,
|
| 1040 |
+
3060,
|
| 1041 |
+
257,
|
| 1042 |
+
25394,
|
| 1043 |
+
393,
|
| 1044 |
+
734,
|
| 1045 |
+
286,
|
| 1046 |
+
262,
|
| 1047 |
+
24240,
|
| 1048 |
+
17008,
|
| 1049 |
+
284,
|
| 1050 |
+
262,
|
| 1051 |
+
11464,
|
| 1052 |
+
13,
|
| 1053 |
+
314,
|
| 1054 |
+
588,
|
| 1055 |
+
284,
|
| 1056 |
+
779,
|
| 1057 |
+
257,
|
| 1058 |
+
15563,
|
| 1059 |
+
611,
|
| 1060 |
+
616,
|
| 1061 |
+
15683,
|
| 1062 |
+
318,
|
| 1063 |
+
319,
|
| 1064 |
+
262,
|
| 1065 |
+
2642,
|
| 1066 |
+
1735,
|
| 1067 |
+
355,
|
| 1068 |
+
326,
|
| 1069 |
+
481,
|
| 1070 |
+
2728,
|
| 1071 |
+
616,
|
| 1072 |
+
4168,
|
| 1073 |
+
284,
|
| 1074 |
+
2270,
|
| 1075 |
+
866,
|
| 1076 |
+
2952,
|
| 1077 |
+
706,
|
| 1078 |
+
4375,
|
| 1079 |
+
517,
|
| 1080 |
+
1660,
|
| 1081 |
+
621,
|
| 1082 |
+
3306,
|
| 1083 |
+
0,
|
| 1084 |
+
198,
|
| 1085 |
+
198,
|
| 1086 |
+
20,
|
| 1087 |
+
15729,
|
| 1088 |
+
2293,
|
| 1089 |
+
10801,
|
| 1090 |
+
477,
|
| 1091 |
+
534,
|
| 1092 |
+
24240,
|
| 1093 |
+
17008,
|
| 1094 |
+
9391,
|
| 1095 |
+
345,
|
| 1096 |
+
815,
|
| 1097 |
+
307,
|
| 1098 |
+
1498,
|
| 1099 |
+
284,
|
| 1100 |
+
651,
|
| 1101 |
+
5755,
|
| 1102 |
+
572,
|
| 1103 |
+
428,
|
| 1104 |
+
640,
|
| 1105 |
+
416,
|
| 1106 |
+
2391,
|
| 1107 |
+
12591,
|
| 1108 |
+
510,
|
| 1109 |
+
262,
|
| 1110 |
+
4220,
|
| 1111 |
+
7679,
|
| 1112 |
+
351,
|
| 1113 |
+
617,
|
| 1114 |
+
517,
|
| 1115 |
+
20236,
|
| 1116 |
+
4283,
|
| 1117 |
+
981,
|
| 1118 |
+
262,
|
| 1119 |
+
5637,
|
| 1120 |
+
11710,
|
| 1121 |
+
3793,
|
| 1122 |
+
2705,
|
| 1123 |
+
523,
|
| 1124 |
+
484,
|
| 1125 |
+
460,
|
| 1126 |
+
447,
|
| 1127 |
+
247,
|
| 1128 |
+
83
|
| 1129 |
+
],
|
| 1130 |
+
"seconds": 2.744,
|
| 1131 |
+
"stop_reason": "token_limit",
|
| 1132 |
+
"record_id": "pretrained/recipe"
|
| 1133 |
+
},
|
| 1134 |
+
{
|
| 1135 |
+
"id": "fox",
|
| 1136 |
+
"prompt": "The little fox stopped at the edge of the forest. In the distance,",
|
| 1137 |
+
"answer": " he was no more than a few feet away. He stopped at a grassy part of the forest, where a group of herding dogs stood on one side and another on some other with their heads down in front from high above them; a little child came to his rescue after they had caught the wolf. They were now moving around a bit closer by the time he heard the loud banging in the trees as it approached him.[11]\n\nThe fox then stepped out into the water below that afternoon upon its death, which was not too long prior when the animal made contact first with the animal itself but quickly returned towards it again for a second time through the hole. It followed this process until it reached the middle of a narrow river near the village, where it went back under to being a",
|
| 1138 |
+
"token_ids": [
|
| 1139 |
+
339,
|
| 1140 |
+
373,
|
| 1141 |
+
645,
|
| 1142 |
+
517,
|
| 1143 |
+
621,
|
| 1144 |
+
257,
|
| 1145 |
+
1178,
|
| 1146 |
+
3625,
|
| 1147 |
+
1497,
|
| 1148 |
+
13,
|
| 1149 |
+
679,
|
| 1150 |
+
5025,
|
| 1151 |
+
379,
|
| 1152 |
+
257,
|
| 1153 |
+
8701,
|
| 1154 |
+
88,
|
| 1155 |
+
636,
|
| 1156 |
+
286,
|
| 1157 |
+
262,
|
| 1158 |
+
8222,
|
| 1159 |
+
11,
|
| 1160 |
+
810,
|
| 1161 |
+
257,
|
| 1162 |
+
1448,
|
| 1163 |
+
286,
|
| 1164 |
+
607,
|
| 1165 |
+
12083,
|
| 1166 |
+
6844,
|
| 1167 |
+
6204,
|
| 1168 |
+
319,
|
| 1169 |
+
530,
|
| 1170 |
+
1735,
|
| 1171 |
+
290,
|
| 1172 |
+
1194,
|
| 1173 |
+
319,
|
| 1174 |
+
617,
|
| 1175 |
+
584,
|
| 1176 |
+
351,
|
| 1177 |
+
511,
|
| 1178 |
+
6665,
|
| 1179 |
+
866,
|
| 1180 |
+
287,
|
| 1181 |
+
2166,
|
| 1182 |
+
422,
|
| 1183 |
+
1029,
|
| 1184 |
+
2029,
|
| 1185 |
+
606,
|
| 1186 |
+
26,
|
| 1187 |
+
257,
|
| 1188 |
+
1310,
|
| 1189 |
+
1200,
|
| 1190 |
+
1625,
|
| 1191 |
+
284,
|
| 1192 |
+
465,
|
| 1193 |
+
9992,
|
| 1194 |
+
706,
|
| 1195 |
+
484,
|
| 1196 |
+
550,
|
| 1197 |
+
4978,
|
| 1198 |
+
262,
|
| 1199 |
+
17481,
|
| 1200 |
+
13,
|
| 1201 |
+
1119,
|
| 1202 |
+
547,
|
| 1203 |
+
783,
|
| 1204 |
+
3867,
|
| 1205 |
+
1088,
|
| 1206 |
+
257,
|
| 1207 |
+
1643,
|
| 1208 |
+
5699,
|
| 1209 |
+
416,
|
| 1210 |
+
262,
|
| 1211 |
+
640,
|
| 1212 |
+
339,
|
| 1213 |
+
2982,
|
| 1214 |
+
262,
|
| 1215 |
+
7812,
|
| 1216 |
+
42028,
|
| 1217 |
+
287,
|
| 1218 |
+
262,
|
| 1219 |
+
7150,
|
| 1220 |
+
355,
|
| 1221 |
+
340,
|
| 1222 |
+
10448,
|
| 1223 |
+
683,
|
| 1224 |
+
3693,
|
| 1225 |
+
1157,
|
| 1226 |
+
60,
|
| 1227 |
+
198,
|
| 1228 |
+
198,
|
| 1229 |
+
464,
|
| 1230 |
+
21831,
|
| 1231 |
+
788,
|
| 1232 |
+
10764,
|
| 1233 |
+
503,
|
| 1234 |
+
656,
|
| 1235 |
+
262,
|
| 1236 |
+
1660,
|
| 1237 |
+
2174,
|
| 1238 |
+
326,
|
| 1239 |
+
6672,
|
| 1240 |
+
2402,
|
| 1241 |
+
663,
|
| 1242 |
+
1918,
|
| 1243 |
+
11,
|
| 1244 |
+
543,
|
| 1245 |
+
373,
|
| 1246 |
+
407,
|
| 1247 |
+
1165,
|
| 1248 |
+
890,
|
| 1249 |
+
3161,
|
| 1250 |
+
618,
|
| 1251 |
+
262,
|
| 1252 |
+
5044,
|
| 1253 |
+
925,
|
| 1254 |
+
2800,
|
| 1255 |
+
717,
|
| 1256 |
+
351,
|
| 1257 |
+
262,
|
| 1258 |
+
5044,
|
| 1259 |
+
2346,
|
| 1260 |
+
475,
|
| 1261 |
+
2952,
|
| 1262 |
+
4504,
|
| 1263 |
+
3371,
|
| 1264 |
+
340,
|
| 1265 |
+
757,
|
| 1266 |
+
329,
|
| 1267 |
+
257,
|
| 1268 |
+
1218,
|
| 1269 |
+
640,
|
| 1270 |
+
832,
|
| 1271 |
+
262,
|
| 1272 |
+
7604,
|
| 1273 |
+
13,
|
| 1274 |
+
632,
|
| 1275 |
+
3940,
|
| 1276 |
+
428,
|
| 1277 |
+
1429,
|
| 1278 |
+
1566,
|
| 1279 |
+
340,
|
| 1280 |
+
4251,
|
| 1281 |
+
262,
|
| 1282 |
+
3504,
|
| 1283 |
+
286,
|
| 1284 |
+
257,
|
| 1285 |
+
7135,
|
| 1286 |
+
7850,
|
| 1287 |
+
1474,
|
| 1288 |
+
262,
|
| 1289 |
+
7404,
|
| 1290 |
+
11,
|
| 1291 |
+
810,
|
| 1292 |
+
340,
|
| 1293 |
+
1816,
|
| 1294 |
+
736,
|
| 1295 |
+
739,
|
| 1296 |
+
284,
|
| 1297 |
+
852,
|
| 1298 |
+
257
|
| 1299 |
+
],
|
| 1300 |
+
"seconds": 2.741,
|
| 1301 |
+
"stop_reason": "token_limit",
|
| 1302 |
+
"record_id": "pretrained/fox"
|
| 1303 |
+
},
|
| 1304 |
+
{
|
| 1305 |
+
"id": "machine",
|
| 1306 |
+
"prompt": "The most important thing I learned from building my first machine was that",
|
| 1307 |
+
"answer": " you can be crazy if something happens, and that’s exactly why it took me so long to figure out what the heck I needed.\n\nI wanted a machine with a custom interface like this was my best hope of getting into a high-performance hardware business in the first place! So instead we decided on a machine for a new kind: an ARM processor, a USB keyboard controller (like this one) based tablet device — which was a key part both of our decision making process about building the machine from scratch through prototyping.\n\nThe last thing anyone has ever asked before is “what were your challenges?” It takes many hours just looking at a project list every day until you know how everything works together back home right now when there are no machines available yet or",
|
| 1308 |
+
"token_ids": [
|
| 1309 |
+
345,
|
| 1310 |
+
460,
|
| 1311 |
+
307,
|
| 1312 |
+
7165,
|
| 1313 |
+
611,
|
| 1314 |
+
1223,
|
| 1315 |
+
4325,
|
| 1316 |
+
11,
|
| 1317 |
+
290,
|
| 1318 |
+
326,
|
| 1319 |
+
447,
|
| 1320 |
+
247,
|
| 1321 |
+
82,
|
| 1322 |
+
3446,
|
| 1323 |
+
1521,
|
| 1324 |
+
340,
|
| 1325 |
+
1718,
|
| 1326 |
+
502,
|
| 1327 |
+
523,
|
| 1328 |
+
890,
|
| 1329 |
+
284,
|
| 1330 |
+
3785,
|
| 1331 |
+
503,
|
| 1332 |
+
644,
|
| 1333 |
+
262,
|
| 1334 |
+
22574,
|
| 1335 |
+
314,
|
| 1336 |
+
2622,
|
| 1337 |
+
13,
|
| 1338 |
+
198,
|
| 1339 |
+
198,
|
| 1340 |
+
40,
|
| 1341 |
+
2227,
|
| 1342 |
+
257,
|
| 1343 |
+
4572,
|
| 1344 |
+
351,
|
| 1345 |
+
257,
|
| 1346 |
+
2183,
|
| 1347 |
+
7071,
|
| 1348 |
+
588,
|
| 1349 |
+
428,
|
| 1350 |
+
373,
|
| 1351 |
+
616,
|
| 1352 |
+
1266,
|
| 1353 |
+
2911,
|
| 1354 |
+
286,
|
| 1355 |
+
1972,
|
| 1356 |
+
656,
|
| 1357 |
+
257,
|
| 1358 |
+
1029,
|
| 1359 |
+
12,
|
| 1360 |
+
26585,
|
| 1361 |
+
6890,
|
| 1362 |
+
1597,
|
| 1363 |
+
287,
|
| 1364 |
+
262,
|
| 1365 |
+
717,
|
| 1366 |
+
1295,
|
| 1367 |
+
0,
|
| 1368 |
+
1406,
|
| 1369 |
+
2427,
|
| 1370 |
+
356,
|
| 1371 |
+
3066,
|
| 1372 |
+
319,
|
| 1373 |
+
257,
|
| 1374 |
+
4572,
|
| 1375 |
+
329,
|
| 1376 |
+
257,
|
| 1377 |
+
649,
|
| 1378 |
+
1611,
|
| 1379 |
+
25,
|
| 1380 |
+
281,
|
| 1381 |
+
20359,
|
| 1382 |
+
12649,
|
| 1383 |
+
11,
|
| 1384 |
+
257,
|
| 1385 |
+
8450,
|
| 1386 |
+
10586,
|
| 1387 |
+
10444,
|
| 1388 |
+
357,
|
| 1389 |
+
2339,
|
| 1390 |
+
428,
|
| 1391 |
+
530,
|
| 1392 |
+
8,
|
| 1393 |
+
1912,
|
| 1394 |
+
14147,
|
| 1395 |
+
3335,
|
| 1396 |
+
851,
|
| 1397 |
+
543,
|
| 1398 |
+
373,
|
| 1399 |
+
257,
|
| 1400 |
+
1994,
|
| 1401 |
+
636,
|
| 1402 |
+
1111,
|
| 1403 |
+
286,
|
| 1404 |
+
674,
|
| 1405 |
+
2551,
|
| 1406 |
+
1642,
|
| 1407 |
+
1429,
|
| 1408 |
+
546,
|
| 1409 |
+
2615,
|
| 1410 |
+
262,
|
| 1411 |
+
4572,
|
| 1412 |
+
422,
|
| 1413 |
+
12692,
|
| 1414 |
+
832,
|
| 1415 |
+
41369,
|
| 1416 |
+
278,
|
| 1417 |
+
13,
|
| 1418 |
+
198,
|
| 1419 |
+
198,
|
| 1420 |
+
464,
|
| 1421 |
+
938,
|
| 1422 |
+
1517,
|
| 1423 |
+
2687,
|
| 1424 |
+
468,
|
| 1425 |
+
1683,
|
| 1426 |
+
1965,
|
| 1427 |
+
878,
|
| 1428 |
+
318,
|
| 1429 |
+
564,
|
| 1430 |
+
250,
|
| 1431 |
+
10919,
|
| 1432 |
+
547,
|
| 1433 |
+
534,
|
| 1434 |
+
6459,
|
| 1435 |
+
30,
|
| 1436 |
+
447,
|
| 1437 |
+
251,
|
| 1438 |
+
632,
|
| 1439 |
+
2753,
|
| 1440 |
+
867,
|
| 1441 |
+
2250,
|
| 1442 |
+
655,
|
| 1443 |
+
2045,
|
| 1444 |
+
379,
|
| 1445 |
+
257,
|
| 1446 |
+
1628,
|
| 1447 |
+
1351,
|
| 1448 |
+
790,
|
| 1449 |
+
1110,
|
| 1450 |
+
1566,
|
| 1451 |
+
345,
|
| 1452 |
+
760,
|
| 1453 |
+
703,
|
| 1454 |
+
2279,
|
| 1455 |
+
2499,
|
| 1456 |
+
1978,
|
| 1457 |
+
736,
|
| 1458 |
+
1363,
|
| 1459 |
+
826,
|
| 1460 |
+
783,
|
| 1461 |
+
618,
|
| 1462 |
+
612,
|
| 1463 |
+
389,
|
| 1464 |
+
645,
|
| 1465 |
+
8217,
|
| 1466 |
+
1695,
|
| 1467 |
+
1865,
|
| 1468 |
+
393
|
| 1469 |
+
],
|
| 1470 |
+
"seconds": 2.662,
|
| 1471 |
+
"stop_reason": "token_limit",
|
| 1472 |
+
"record_id": "pretrained/machine"
|
| 1473 |
+
},
|
| 1474 |
+
{
|
| 1475 |
+
"id": "library",
|
| 1476 |
+
"prompt": "The town library opened its doors on a rainy morning. Inside, the librarian was preparing",
|
| 1477 |
+
"answer": " for the day’s events, and had just completed some homework. She picked up her copy of Time magazine on the floor to read it over at the library in a lecture class that ended around noon today…\n\nThe author of The New Yorker, Stephen Rabinowitz, was invited by his wife to take part Tuesday night as the editorial director before a panel about the future of journalism — a panel featuring “the best writers from all corners because we know they have something special going into them tonight!” — at the Library of Congress, the home of the Library of Congress, the library where he has lived since 1976 (he’d been working there ever so long). When the panel began, he said: I hope you will join me here with me now when I",
|
| 1478 |
+
"token_ids": [
|
| 1479 |
+
329,
|
| 1480 |
+
262,
|
| 1481 |
+
1110,
|
| 1482 |
+
447,
|
| 1483 |
+
247,
|
| 1484 |
+
82,
|
| 1485 |
+
2995,
|
| 1486 |
+
11,
|
| 1487 |
+
290,
|
| 1488 |
+
550,
|
| 1489 |
+
655,
|
| 1490 |
+
5668,
|
| 1491 |
+
617,
|
| 1492 |
+
26131,
|
| 1493 |
+
13,
|
| 1494 |
+
1375,
|
| 1495 |
+
6497,
|
| 1496 |
+
510,
|
| 1497 |
+
607,
|
| 1498 |
+
4866,
|
| 1499 |
+
286,
|
| 1500 |
+
3862,
|
| 1501 |
+
7093,
|
| 1502 |
+
319,
|
| 1503 |
+
262,
|
| 1504 |
+
4314,
|
| 1505 |
+
284,
|
| 1506 |
+
1100,
|
| 1507 |
+
340,
|
| 1508 |
+
625,
|
| 1509 |
+
379,
|
| 1510 |
+
262,
|
| 1511 |
+
5888,
|
| 1512 |
+
287,
|
| 1513 |
+
257,
|
| 1514 |
+
19143,
|
| 1515 |
+
1398,
|
| 1516 |
+
326,
|
| 1517 |
+
4444,
|
| 1518 |
+
1088,
|
| 1519 |
+
19613,
|
| 1520 |
+
1909,
|
| 1521 |
+
1399,
|
| 1522 |
+
198,
|
| 1523 |
+
198,
|
| 1524 |
+
464,
|
| 1525 |
+
1772,
|
| 1526 |
+
286,
|
| 1527 |
+
383,
|
| 1528 |
+
968,
|
| 1529 |
+
27980,
|
| 1530 |
+
11,
|
| 1531 |
+
7970,
|
| 1532 |
+
371,
|
| 1533 |
+
6014,
|
| 1534 |
+
20951,
|
| 1535 |
+
11,
|
| 1536 |
+
373,
|
| 1537 |
+
9392,
|
| 1538 |
+
416,
|
| 1539 |
+
465,
|
| 1540 |
+
3656,
|
| 1541 |
+
284,
|
| 1542 |
+
1011,
|
| 1543 |
+
636,
|
| 1544 |
+
3431,
|
| 1545 |
+
1755,
|
| 1546 |
+
355,
|
| 1547 |
+
262,
|
| 1548 |
+
13684,
|
| 1549 |
+
3437,
|
| 1550 |
+
878,
|
| 1551 |
+
257,
|
| 1552 |
+
6103,
|
| 1553 |
+
546,
|
| 1554 |
+
262,
|
| 1555 |
+
2003,
|
| 1556 |
+
286,
|
| 1557 |
+
14033,
|
| 1558 |
+
851,
|
| 1559 |
+
257,
|
| 1560 |
+
6103,
|
| 1561 |
+
9593,
|
| 1562 |
+
564,
|
| 1563 |
+
250,
|
| 1564 |
+
1169,
|
| 1565 |
+
1266,
|
| 1566 |
+
8786,
|
| 1567 |
+
422,
|
| 1568 |
+
477,
|
| 1569 |
+
14371,
|
| 1570 |
+
780,
|
| 1571 |
+
356,
|
| 1572 |
+
760,
|
| 1573 |
+
484,
|
| 1574 |
+
423,
|
| 1575 |
+
1223,
|
| 1576 |
+
2041,
|
| 1577 |
+
1016,
|
| 1578 |
+
656,
|
| 1579 |
+
606,
|
| 1580 |
+
9975,
|
| 1581 |
+
0,
|
| 1582 |
+
447,
|
| 1583 |
+
251,
|
| 1584 |
+
851,
|
| 1585 |
+
379,
|
| 1586 |
+
262,
|
| 1587 |
+
10074,
|
| 1588 |
+
286,
|
| 1589 |
+
3162,
|
| 1590 |
+
11,
|
| 1591 |
+
262,
|
| 1592 |
+
1363,
|
| 1593 |
+
286,
|
| 1594 |
+
262,
|
| 1595 |
+
10074,
|
| 1596 |
+
286,
|
| 1597 |
+
3162,
|
| 1598 |
+
11,
|
| 1599 |
+
262,
|
| 1600 |
+
5888,
|
| 1601 |
+
810,
|
| 1602 |
+
339,
|
| 1603 |
+
468,
|
| 1604 |
+
5615,
|
| 1605 |
+
1201,
|
| 1606 |
+
15408,
|
| 1607 |
+
357,
|
| 1608 |
+
258,
|
| 1609 |
+
447,
|
| 1610 |
+
247,
|
| 1611 |
+
67,
|
| 1612 |
+
587,
|
| 1613 |
+
1762,
|
| 1614 |
+
612,
|
| 1615 |
+
1683,
|
| 1616 |
+
523,
|
| 1617 |
+
890,
|
| 1618 |
+
737,
|
| 1619 |
+
1649,
|
| 1620 |
+
262,
|
| 1621 |
+
6103,
|
| 1622 |
+
2540,
|
| 1623 |
+
11,
|
| 1624 |
+
339,
|
| 1625 |
+
531,
|
| 1626 |
+
25,
|
| 1627 |
+
314,
|
| 1628 |
+
2911,
|
| 1629 |
+
345,
|
| 1630 |
+
481,
|
| 1631 |
+
4654,
|
| 1632 |
+
502,
|
| 1633 |
+
994,
|
| 1634 |
+
351,
|
| 1635 |
+
502,
|
| 1636 |
+
783,
|
| 1637 |
+
618,
|
| 1638 |
+
314
|
| 1639 |
+
],
|
| 1640 |
+
"seconds": 2.591,
|
| 1641 |
+
"stop_reason": "token_limit",
|
| 1642 |
+
"record_id": "pretrained/library"
|
| 1643 |
+
}
|
| 1644 |
+
]
|
| 1645 |
+
}
|
| 1646 |
+
}
|
| 1647 |
+
}
|
task_finetuning/results/comparison.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
task_finetuning/results/dataset_manifest.json
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"prepared_at_utc": "2026-10-26T05:49:36.901780+00:00",
|
| 3 |
+
"tasks": {
|
| 4 |
+
"sst2": {
|
| 5 |
+
"dataset": "stanfordnlp/sst2",
|
| 6 |
+
"config": "default",
|
| 7 |
+
"metadata_url": "https://datasets-server.huggingface.co/parquet?dataset=stanfordnlp%2Fsst2",
|
| 8 |
+
"splits": {
|
| 9 |
+
"train": {
|
| 10 |
+
"path": "original_workspace/output/gpt1_style/task_finetuning/data/sst2_train.jsonl",
|
| 11 |
+
"rows": 67349,
|
| 12 |
+
"sha256": "75fd14e7a62a5d54d1300996afaef6223ec8c49eecd97f0674d6664489a69daa",
|
| 13 |
+
"label_counts": {
|
| 14 |
+
"0": 29780,
|
| 15 |
+
"1": 37569
|
| 16 |
+
},
|
| 17 |
+
"labelled": true,
|
| 18 |
+
"source_parquets": [
|
| 19 |
+
{
|
| 20 |
+
"url": "https://huggingface.co/datasets/stanfordnlp/sst2/resolve/refs%2Fconvert%2Fparquet/default/train/0000.parquet",
|
| 21 |
+
"path": "original_workspace/output/gpt1_style/task_finetuning/data/source_parquets/sst2/train/000_0000.parquet",
|
| 22 |
+
"bytes": 3110458,
|
| 23 |
+
"sha256": "c7921283b75a42e685f50edecb96798607ea0fcbfd0739ee8975f22c12d55f09"
|
| 24 |
+
}
|
| 25 |
+
]
|
| 26 |
+
},
|
| 27 |
+
"validation": {
|
| 28 |
+
"path": "original_workspace/output/gpt1_style/task_finetuning/data/sst2_validation.jsonl",
|
| 29 |
+
"rows": 872,
|
| 30 |
+
"sha256": "9f7afa11e8d0309f4ea4732c7f11d78fb0f686f48fac5596a06db2ecf496678b",
|
| 31 |
+
"label_counts": {
|
| 32 |
+
"0": 428,
|
| 33 |
+
"1": 444
|
| 34 |
+
},
|
| 35 |
+
"labelled": true,
|
| 36 |
+
"source_parquets": [
|
| 37 |
+
{
|
| 38 |
+
"url": "https://huggingface.co/datasets/stanfordnlp/sst2/resolve/refs%2Fconvert%2Fparquet/default/validation/0000.parquet",
|
| 39 |
+
"path": "original_workspace/output/gpt1_style/task_finetuning/data/source_parquets/sst2/validation/000_0000.parquet",
|
| 40 |
+
"bytes": 72813,
|
| 41 |
+
"sha256": "fb00fe008f6828f86ba2beda8415a4cf5da0c884f21c5f238c87131b5aa19529"
|
| 42 |
+
}
|
| 43 |
+
]
|
| 44 |
+
},
|
| 45 |
+
"test": {
|
| 46 |
+
"path": "original_workspace/output/gpt1_style/task_finetuning/data/sst2_test.jsonl",
|
| 47 |
+
"rows": 1821,
|
| 48 |
+
"sha256": "2b8c1efc9ebea4f988382cc5d91aa3b636bff90e16795d705b2a25ef8b2cfd5b",
|
| 49 |
+
"label_counts": {
|
| 50 |
+
"-1": 1821
|
| 51 |
+
},
|
| 52 |
+
"labelled": false,
|
| 53 |
+
"source_parquets": [
|
| 54 |
+
{
|
| 55 |
+
"url": "https://huggingface.co/datasets/stanfordnlp/sst2/resolve/refs%2Fconvert%2Fparquet/default/test/0000.parquet",
|
| 56 |
+
"path": "original_workspace/output/gpt1_style/task_finetuning/data/source_parquets/sst2/test/000_0000.parquet",
|
| 57 |
+
"bytes": 147787,
|
| 58 |
+
"sha256": "20d27a86c0c59acb746a41a481ebb1fc71edb72d94b5ccee7f23b9041b17adcf"
|
| 59 |
+
}
|
| 60 |
+
]
|
| 61 |
+
}
|
| 62 |
+
}
|
| 63 |
+
},
|
| 64 |
+
"race_middle": {
|
| 65 |
+
"dataset": "ehovy/race",
|
| 66 |
+
"config": "middle",
|
| 67 |
+
"metadata_url": "https://datasets-server.huggingface.co/parquet?dataset=ehovy%2Frace",
|
| 68 |
+
"splits": {
|
| 69 |
+
"train": {
|
| 70 |
+
"path": "original_workspace/output/gpt1_style/task_finetuning/data/race_middle_train.jsonl",
|
| 71 |
+
"rows": 25421,
|
| 72 |
+
"sha256": "ed805e44773ff5856211f0713270222c8ca681e84c43b549cdecfbef97a46652",
|
| 73 |
+
"label_counts": {
|
| 74 |
+
"0": 5445,
|
| 75 |
+
"1": 6705,
|
| 76 |
+
"2": 7006,
|
| 77 |
+
"3": 6265
|
| 78 |
+
},
|
| 79 |
+
"labelled": true,
|
| 80 |
+
"source_parquets": [
|
| 81 |
+
{
|
| 82 |
+
"url": "https://huggingface.co/datasets/ehovy/race/resolve/refs%2Fconvert%2Fparquet/middle/train/0000.parquet",
|
| 83 |
+
"path": "original_workspace/output/gpt1_style/task_finetuning/data/source_parquets/race_middle/train/000_0000.parquet",
|
| 84 |
+
"bytes": 6969931,
|
| 85 |
+
"sha256": "33a9d28467c326b13684bddd651a3bdc057aea61b35f93258c95efd76a435f63"
|
| 86 |
+
}
|
| 87 |
+
]
|
| 88 |
+
},
|
| 89 |
+
"validation": {
|
| 90 |
+
"path": "original_workspace/output/gpt1_style/task_finetuning/data/race_middle_validation.jsonl",
|
| 91 |
+
"rows": 1436,
|
| 92 |
+
"sha256": "385d417aaddbc25038d7bd4afbf9f385d4e3ae74bff532397ba938b0fbf0f622",
|
| 93 |
+
"label_counts": {
|
| 94 |
+
"0": 311,
|
| 95 |
+
"1": 384,
|
| 96 |
+
"2": 386,
|
| 97 |
+
"3": 355
|
| 98 |
+
},
|
| 99 |
+
"labelled": true,
|
| 100 |
+
"source_parquets": [
|
| 101 |
+
{
|
| 102 |
+
"url": "https://huggingface.co/datasets/ehovy/race/resolve/refs%2Fconvert%2Fparquet/middle/validation/0000.parquet",
|
| 103 |
+
"path": "original_workspace/output/gpt1_style/task_finetuning/data/source_parquets/race_middle/validation/000_0000.parquet",
|
| 104 |
+
"bytes": 406940,
|
| 105 |
+
"sha256": "c8ede76683827fc6f5219643626d171c2556ee6b47193b192f399c2f7c4e60a0"
|
| 106 |
+
}
|
| 107 |
+
]
|
| 108 |
+
},
|
| 109 |
+
"test": {
|
| 110 |
+
"path": "original_workspace/output/gpt1_style/task_finetuning/data/race_middle_test.jsonl",
|
| 111 |
+
"rows": 1436,
|
| 112 |
+
"sha256": "dd9756cf41d600260b59064c2b10395affb53d769a4bf71017ca8dc6ceb005d9",
|
| 113 |
+
"label_counts": {
|
| 114 |
+
"0": 309,
|
| 115 |
+
"1": 388,
|
| 116 |
+
"2": 374,
|
| 117 |
+
"3": 365
|
| 118 |
+
},
|
| 119 |
+
"labelled": true,
|
| 120 |
+
"source_parquets": [
|
| 121 |
+
{
|
| 122 |
+
"url": "https://huggingface.co/datasets/ehovy/race/resolve/refs%2Fconvert%2Fparquet/middle/test/0000.parquet",
|
| 123 |
+
"path": "original_workspace/output/gpt1_style/task_finetuning/data/source_parquets/race_middle/test/000_0000.parquet",
|
| 124 |
+
"bytes": 404725,
|
| 125 |
+
"sha256": "4ea3fb86ab6bbf8746dd0eb0ac1961a1fdbd1214d6d5a901c37b0cad5b5c5971"
|
| 126 |
+
}
|
| 127 |
+
]
|
| 128 |
+
}
|
| 129 |
+
}
|
| 130 |
+
}
|
| 131 |
+
},
|
| 132 |
+
"split_policy": {
|
| 133 |
+
"sst2": "Train on official train; validation is the only public labelled holdout. Official test labels are hidden (-1).",
|
| 134 |
+
"race_middle": "Train on official middle/train; select using middle/validation; evaluate once on middle/test."
|
| 135 |
+
},
|
| 136 |
+
"normalization": "Exact input strings retained; sentiment labels are 0=negative,1=positive; RACE A/B/C/D mapped to integer 0/1/2/3; no examples generated or reassigned."
|
| 137 |
+
}
|
task_finetuning/results/encoding_summary.json
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"prepared_at_utc": "2026-10-26T05:53:07.208785+00:00",
|
| 3 |
+
"encoder": "Exact bundled GPT-2/r50k via wiggle_assistant.get_encoder; ordinary-string encoding only",
|
| 4 |
+
"tokenizer_source": "original_workspace/WiggleGPT-Gosport/wiggle_assistant.py",
|
| 5 |
+
"tokenizer_source_sha256": "fc10c18e3cd43986437c0f2ecde5a6b1c167423d9db98243e6f8c7e393711791",
|
| 6 |
+
"merge_table_sha256": "306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930",
|
| 7 |
+
"task_markers": {
|
| 8 |
+
"START": 50257,
|
| 9 |
+
"DELIM": 50258,
|
| 10 |
+
"CLS": 50259,
|
| 11 |
+
"PAD": 50256
|
| 12 |
+
},
|
| 13 |
+
"limit": 512,
|
| 14 |
+
"cpu_threads": 2,
|
| 15 |
+
"splits": {
|
| 16 |
+
"sst2_train": {
|
| 17 |
+
"task": "sst2",
|
| 18 |
+
"split": "train",
|
| 19 |
+
"examples": 67349,
|
| 20 |
+
"choices": 1,
|
| 21 |
+
"source_jsonl": "original_workspace/output/gpt1_style/task_finetuning/data/sst2_train.jsonl",
|
| 22 |
+
"source_sha256": "75fd14e7a62a5d54d1300996afaef6223ec8c49eecd97f0674d6664489a69daa",
|
| 23 |
+
"pad_length": 80,
|
| 24 |
+
"max_sequence_length": 67,
|
| 25 |
+
"min_sequence_length": 4,
|
| 26 |
+
"max_untruncated_length": 67,
|
| 27 |
+
"truncation": {
|
| 28 |
+
"examples_with_truncation": 0,
|
| 29 |
+
"candidate_sequences_with_truncation": 0,
|
| 30 |
+
"total_tokens_removed": 0,
|
| 31 |
+
"max_tokens_removed_per_example": 0
|
| 32 |
+
},
|
| 33 |
+
"context_policy": "[START] + review first 510 tokens + [CLS]; right-pad to the split maximum rounded up to a multiple of 16 (minimum 16).",
|
| 34 |
+
"input_ids_dtype": "int32",
|
| 35 |
+
"lengths_dtype": "int64",
|
| 36 |
+
"labels_dtype": "int64",
|
| 37 |
+
"cache_path": "original_workspace/output/gpt1_style/task_finetuning/data/sst2_train.pt",
|
| 38 |
+
"cache_sha256": "eacd114353b3cd0a0a89490a2d81e54f4ddc95d65cd74efb143b50c7b5581e57",
|
| 39 |
+
"cache_bytes": 24371735
|
| 40 |
+
},
|
| 41 |
+
"sst2_validation": {
|
| 42 |
+
"task": "sst2",
|
| 43 |
+
"split": "validation",
|
| 44 |
+
"examples": 872,
|
| 45 |
+
"choices": 1,
|
| 46 |
+
"source_jsonl": "original_workspace/output/gpt1_style/task_finetuning/data/sst2_validation.jsonl",
|
| 47 |
+
"source_sha256": "9f7afa11e8d0309f4ea4732c7f11d78fb0f686f48fac5596a06db2ecf496678b",
|
| 48 |
+
"pad_length": 64,
|
| 49 |
+
"max_sequence_length": 63,
|
| 50 |
+
"min_sequence_length": 5,
|
| 51 |
+
"max_untruncated_length": 63,
|
| 52 |
+
"truncation": {
|
| 53 |
+
"examples_with_truncation": 0,
|
| 54 |
+
"candidate_sequences_with_truncation": 0,
|
| 55 |
+
"total_tokens_removed": 0,
|
| 56 |
+
"max_tokens_removed_per_example": 0
|
| 57 |
+
},
|
| 58 |
+
"context_policy": "[START] + review first 510 tokens + [CLS]; right-pad to the split maximum rounded up to a multiple of 16 (minimum 16).",
|
| 59 |
+
"input_ids_dtype": "int32",
|
| 60 |
+
"lengths_dtype": "int64",
|
| 61 |
+
"labels_dtype": "int64",
|
| 62 |
+
"cache_path": "original_workspace/output/gpt1_style/task_finetuning/data/sst2_validation.pt",
|
| 63 |
+
"cache_sha256": "22eed8372c39df075210f60de11263bfd6f8dd17cc934901e0d1f807410bf943",
|
| 64 |
+
"cache_bytes": 264900
|
| 65 |
+
},
|
| 66 |
+
"sst2_test": {
|
| 67 |
+
"task": "sst2",
|
| 68 |
+
"split": "test",
|
| 69 |
+
"examples": 1821,
|
| 70 |
+
"choices": 1,
|
| 71 |
+
"source_jsonl": "original_workspace/output/gpt1_style/task_finetuning/data/sst2_test.jsonl",
|
| 72 |
+
"source_sha256": "2b8c1efc9ebea4f988382cc5d91aa3b636bff90e16795d705b2a25ef8b2cfd5b",
|
| 73 |
+
"pad_length": 64,
|
| 74 |
+
"max_sequence_length": 63,
|
| 75 |
+
"min_sequence_length": 5,
|
| 76 |
+
"max_untruncated_length": 63,
|
| 77 |
+
"truncation": {
|
| 78 |
+
"examples_with_truncation": 0,
|
| 79 |
+
"candidate_sequences_with_truncation": 0,
|
| 80 |
+
"total_tokens_removed": 0,
|
| 81 |
+
"max_tokens_removed_per_example": 0
|
| 82 |
+
},
|
| 83 |
+
"context_policy": "[START] + review first 510 tokens + [CLS]; right-pad to the split maximum rounded up to a multiple of 16 (minimum 16).",
|
| 84 |
+
"input_ids_dtype": "int32",
|
| 85 |
+
"lengths_dtype": "int64",
|
| 86 |
+
"labels_dtype": "int64",
|
| 87 |
+
"cache_path": "original_workspace/output/gpt1_style/task_finetuning/data/sst2_test.pt",
|
| 88 |
+
"cache_sha256": "577a4c443c0bf54dd01c12d63c0b5a485d455256d741a740363791db6298cdd0",
|
| 89 |
+
"cache_bytes": 540302
|
| 90 |
+
},
|
| 91 |
+
"race_middle_train": {
|
| 92 |
+
"task": "race_middle",
|
| 93 |
+
"split": "train",
|
| 94 |
+
"examples": 25421,
|
| 95 |
+
"choices": 4,
|
| 96 |
+
"source_jsonl": "original_workspace/output/gpt1_style/task_finetuning/data/race_middle_train.jsonl",
|
| 97 |
+
"source_sha256": "ed805e44773ff5856211f0713270222c8ca681e84c43b549cdecfbef97a46652",
|
| 98 |
+
"pad_length": 512,
|
| 99 |
+
"max_sequence_length": 512,
|
| 100 |
+
"min_sequence_length": 20,
|
| 101 |
+
"max_untruncated_length": 673,
|
| 102 |
+
"truncation": {
|
| 103 |
+
"examples_with_truncation": 210,
|
| 104 |
+
"candidate_sequences_with_truncation": 840,
|
| 105 |
+
"total_tokens_removed": 10052,
|
| 106 |
+
"max_tokens_removed_per_example": 161
|
| 107 |
+
},
|
| 108 |
+
"context_policy": "[START] + shared article prefix + [DELIM] + complete question + [DELIM] + complete answer + [CLS]; all four choices retain the same article prefix, budgeted using the longest answer; right-pad to 512.",
|
| 109 |
+
"input_ids_dtype": "int32",
|
| 110 |
+
"lengths_dtype": "int64",
|
| 111 |
+
"labels_dtype": "int64",
|
| 112 |
+
"cache_path": "original_workspace/output/gpt1_style/task_finetuning/data/race_middle_train.pt",
|
| 113 |
+
"cache_sha256": "5494c6d1a4fcca3651e819680e094693db032a36991d4b1341b108d7d0c5a797",
|
| 114 |
+
"cache_bytes": 209874966
|
| 115 |
+
},
|
| 116 |
+
"race_middle_validation": {
|
| 117 |
+
"task": "race_middle",
|
| 118 |
+
"split": "validation",
|
| 119 |
+
"examples": 1436,
|
| 120 |
+
"choices": 4,
|
| 121 |
+
"source_jsonl": "original_workspace/output/gpt1_style/task_finetuning/data/race_middle_validation.jsonl",
|
| 122 |
+
"source_sha256": "385d417aaddbc25038d7bd4afbf9f385d4e3ae74bff532397ba938b0fbf0f622",
|
| 123 |
+
"pad_length": 512,
|
| 124 |
+
"max_sequence_length": 512,
|
| 125 |
+
"min_sequence_length": 65,
|
| 126 |
+
"max_untruncated_length": 530,
|
| 127 |
+
"truncation": {
|
| 128 |
+
"examples_with_truncation": 9,
|
| 129 |
+
"candidate_sequences_with_truncation": 36,
|
| 130 |
+
"total_tokens_removed": 71,
|
| 131 |
+
"max_tokens_removed_per_example": 18
|
| 132 |
+
},
|
| 133 |
+
"context_policy": "[START] + shared article prefix + [DELIM] + complete question + [DELIM] + complete answer + [CLS]; all four choices retain the same article prefix, budgeted using the longest answer; right-pad to 512.",
|
| 134 |
+
"input_ids_dtype": "int32",
|
| 135 |
+
"lengths_dtype": "int64",
|
| 136 |
+
"labels_dtype": "int64",
|
| 137 |
+
"cache_path": "original_workspace/output/gpt1_style/task_finetuning/data/race_middle_validation.pt",
|
| 138 |
+
"cache_sha256": "fb2db9a134abcf138b093d251172c6ffde4e58db60a0c2a56657956ed9afeae6",
|
| 139 |
+
"cache_bytes": 11858115
|
| 140 |
+
},
|
| 141 |
+
"race_middle_test": {
|
| 142 |
+
"task": "race_middle",
|
| 143 |
+
"split": "test",
|
| 144 |
+
"examples": 1436,
|
| 145 |
+
"choices": 4,
|
| 146 |
+
"source_jsonl": "original_workspace/output/gpt1_style/task_finetuning/data/race_middle_test.jsonl",
|
| 147 |
+
"source_sha256": "dd9756cf41d600260b59064c2b10395affb53d769a4bf71017ca8dc6ceb005d9",
|
| 148 |
+
"pad_length": 512,
|
| 149 |
+
"max_sequence_length": 512,
|
| 150 |
+
"min_sequence_length": 41,
|
| 151 |
+
"max_untruncated_length": 565,
|
| 152 |
+
"truncation": {
|
| 153 |
+
"examples_with_truncation": 14,
|
| 154 |
+
"candidate_sequences_with_truncation": 56,
|
| 155 |
+
"total_tokens_removed": 531,
|
| 156 |
+
"max_tokens_removed_per_example": 53
|
| 157 |
+
},
|
| 158 |
+
"context_policy": "[START] + shared article prefix + [DELIM] + complete question + [DELIM] + complete answer + [CLS]; all four choices retain the same article prefix, budgeted using the longest answer; right-pad to 512.",
|
| 159 |
+
"input_ids_dtype": "int32",
|
| 160 |
+
"lengths_dtype": "int64",
|
| 161 |
+
"labels_dtype": "int64",
|
| 162 |
+
"cache_path": "original_workspace/output/gpt1_style/task_finetuning/data/race_middle_test.pt",
|
| 163 |
+
"cache_sha256": "7d5577758ee2fbd2429edc03669d2f539568158d6658683c8f872ce400375e56",
|
| 164 |
+
"cache_bytes": 11857997
|
| 165 |
+
}
|
| 166 |
+
}
|
| 167 |
+
}
|
task_finetuning/results/manifest.json
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"release_date": "2026-10-26",
|
| 3 |
+
"scope": "Portable task-fine-tuning source, documentation and recorded completed results; model weights and source corpus excluded.",
|
| 4 |
+
"hash_algorithm": "sha256",
|
| 5 |
+
"files": [
|
| 6 |
+
{
|
| 7 |
+
"path": ".gitignore",
|
| 8 |
+
"bytes": 83,
|
| 9 |
+
"sha256": "3526090e3b9dc17c59f7e0d6ae5dbcda1de7a6bda18089664735cb6a690b6041"
|
| 10 |
+
},
|
| 11 |
+
{
|
| 12 |
+
"path": "data/.gitkeep",
|
| 13 |
+
"bytes": 1,
|
| 14 |
+
"sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b"
|
| 15 |
+
},
|
| 16 |
+
{
|
| 17 |
+
"path": "encode_task_data.py",
|
| 18 |
+
"bytes": 6919,
|
| 19 |
+
"sha256": "a34ff2961aed6885edaabea57698d6f335ea133ca8cbc31dc1cd7ae6620db656"
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"path": "evaluate_tasks.py",
|
| 23 |
+
"bytes": 3931,
|
| 24 |
+
"sha256": "dd7f6aafd74864781f9d3a1d48d93afcfbb2a66d4890083aba7742cf93c492bd"
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"path": "native/model_bio.py",
|
| 28 |
+
"bytes": 22768,
|
| 29 |
+
"sha256": "a28201b8194941a75d3ca2e0812e6d2d0eef1b2656cc0c9f8fd0ca65bf96b28b"
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"path": "prepare_data.py",
|
| 33 |
+
"bytes": 5877,
|
| 34 |
+
"sha256": "ded7db958c65fc2ab4bc65d941e3d0dec1f7fe6d82e854f1ef05fdb955563d45"
|
| 35 |
+
},
|
| 36 |
+
{
|
| 37 |
+
"path": "README.md",
|
| 38 |
+
"bytes": 8842,
|
| 39 |
+
"sha256": "40d4095b69809172497b85f54a6d01f60040eaff10c86cc0b92a188f518ce254"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"path": "requirements.txt",
|
| 43 |
+
"bytes": 197,
|
| 44 |
+
"sha256": "4d4f6bbb4eeb6313107c263be41782dda4126b3111687c97653f6b1eee305757"
|
| 45 |
+
},
|
| 46 |
+
{
|
| 47 |
+
"path": "results/comparison.json",
|
| 48 |
+
"bytes": 2011762,
|
| 49 |
+
"sha256": "bc1ca4d680729e5b31596c439b2747973f761c6ff59c6d82afd5b97b0f2f6fb5"
|
| 50 |
+
},
|
| 51 |
+
{
|
| 52 |
+
"path": "results/dataset_manifest.json",
|
| 53 |
+
"bytes": 6059,
|
| 54 |
+
"sha256": "cac78408eede4920cd81b254076452cf7fac0bbcac11e745bb8e84faa2e4cad7"
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"path": "results/encoding_summary.json",
|
| 58 |
+
"bytes": 7687,
|
| 59 |
+
"sha256": "fc19f57cd388770c23c4c70c1387dbb2b7ac1d94b7454e42a158a088056c7b77"
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"path": "results/race_middle_metrics.jsonl",
|
| 63 |
+
"bytes": 41545,
|
| 64 |
+
"sha256": "1c9317edef4001cfd08d8596f62c078e0ba2c4b853cf7d808dd09fb43a428109"
|
| 65 |
+
},
|
| 66 |
+
{
|
| 67 |
+
"path": "results/race_middle_provenance.json",
|
| 68 |
+
"bytes": 5766,
|
| 69 |
+
"sha256": "b1d3d6e88288d499739a685859f915a419f5c19835180a3d14ae2e3e9228dbdd"
|
| 70 |
+
},
|
| 71 |
+
{
|
| 72 |
+
"path": "results/race_middle_resume_provenance.json",
|
| 73 |
+
"bytes": 1395,
|
| 74 |
+
"sha256": "fdddebde55b6aadbabca3eb52c4836d6d0b52bc5818f7a480f57db352909ffe5"
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"path": "results/sst2_metrics.jsonl",
|
| 78 |
+
"bytes": 93890,
|
| 79 |
+
"sha256": "cd4687d15c9961226589b9af0fb120d0b5a0d5fc842805613a2c68bb40a9ba10"
|
| 80 |
+
},
|
| 81 |
+
{
|
| 82 |
+
"path": "results/sst2_provenance.json",
|
| 83 |
+
"bytes": 3517,
|
| 84 |
+
"sha256": "d0d0e3725e2e1a4850ffe4eba35500bfa5472db7d01a743b2d28e59a1187e1bf"
|
| 85 |
+
},
|
| 86 |
+
{
|
| 87 |
+
"path": "results/train_tasks_original_source.txt",
|
| 88 |
+
"bytes": 25970,
|
| 89 |
+
"sha256": "550f68528e3dbe630cb7f0ad29860d8bafb77767832819dab86ea598a29ad520"
|
| 90 |
+
},
|
| 91 |
+
{
|
| 92 |
+
"path": "results/train_tasks_v2_original_source.txt",
|
| 93 |
+
"bytes": 32940,
|
| 94 |
+
"sha256": "aaa80085a75147c02945e7e09bcdd0d18c929a79a12ffce2a830fa29c5d81cfe"
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"path": "results/train_tasks_v3_original_source.txt",
|
| 98 |
+
"bytes": 35247,
|
| 99 |
+
"sha256": "080b06025459b59c2dddb2be046f28c5906e40915dbbb325a8c64642511d5517"
|
| 100 |
+
},
|
| 101 |
+
{
|
| 102 |
+
"path": "results/WiggleGPT_Generation_Record.json",
|
| 103 |
+
"bytes": 49589,
|
| 104 |
+
"sha256": "c6ddd0ddda5be70123183de255f692fc12a1ea3ea1322df6939683ca978a62b6"
|
| 105 |
+
},
|
| 106 |
+
{
|
| 107 |
+
"path": "task_runtime.py",
|
| 108 |
+
"bytes": 4263,
|
| 109 |
+
"sha256": "220b2530cf05fabd313a58ca56059bfcc3332a77b3ee1cd374510996fc22c227"
|
| 110 |
+
},
|
| 111 |
+
{
|
| 112 |
+
"path": "tokenizer/gpt2.tiktoken",
|
| 113 |
+
"bytes": 835554,
|
| 114 |
+
"sha256": "306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930"
|
| 115 |
+
},
|
| 116 |
+
{
|
| 117 |
+
"path": "train_config.json",
|
| 118 |
+
"bytes": 892,
|
| 119 |
+
"sha256": "93303a4fa1a527dcf8390eaec4bb9a557d47e30dd42245e85b39658fd3a7e5d2"
|
| 120 |
+
},
|
| 121 |
+
{
|
| 122 |
+
"path": "train_tasks.py",
|
| 123 |
+
"bytes": 26998,
|
| 124 |
+
"sha256": "b6dc440f26f74f58439640432828e7c1992d2f056896d554ad7fb647beb3c962"
|
| 125 |
+
},
|
| 126 |
+
{
|
| 127 |
+
"path": "train_tasks_v3.py",
|
| 128 |
+
"bytes": 36077,
|
| 129 |
+
"sha256": "57f66db8fe63d254a430e5d97c3757311fdc0e74ce40f9dbc76c56ddd696cbbb"
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"path": "WiggleGPT_Demonstration.pdf",
|
| 133 |
+
"bytes": 2187032,
|
| 134 |
+
"sha256": "032920d6249b0e392012bfb6da0f8e5d34ca1bb173f2c75b2bb902e580330ad1"
|
| 135 |
+
}
|
| 136 |
+
]
|
| 137 |
+
}
|
task_finetuning/results/race_middle_metrics.jsonl
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"event": "start", "step": 0, "train_examples": 25421, "epochs": 3, "total_steps": 2385, "microbatch": 2, "effective_batch": 32, "memory_cap_gib": 9.0, "trainable_parameters": 123698689, "device": "NVIDIA GeForce RTX 5060 Ti", "optimizations": {"version": 2, "training_padding": "CPU epoch plan crops each unchanged microbatch to its longest candidate, rounded up to 16 tokens (minimum 16).", "auxiliary_lm": "Project only valid next-token positions; weight each token by the inverse length of its own candidate, then average all candidates.", "device_synchronization": "One order copy to CPU per epoch; CPU-known output_size avoids synchronization when building packed CUDA indices.", "question_order_and_effective_batch": "Original seeded CUDA randperm, effective batch and microbatch membership retained.", "native_backbone": "Unchanged native model_bio.py and configured gradient checkpointing retained."}, "verify_packed_loss_once": true, "task": "race_middle"}
|
| 2 |
+
{"event": "before", "heldout": "test", "correct": 385, "total": 1436, "accuracy": 0.2681058495821727, "loss": 2.6538400650024414, "gold_label_counts": [309, 388, 374, 365], "majority_accuracy": 0.27019498607242337, "task": "race_middle"}
|
| 3 |
+
{"event": "eval", "step": 0, "epoch": 0, "offset": 0, "train": {"correct": 33, "total": 128, "accuracy": 0.2578125, "loss": 2.7953665256500244, "gold_label_counts": [30, 31, 38, 29], "majority_accuracy": 0.296875}, "validation": {"correct": 380, "total": 1436, "accuracy": 0.2646239554317549, "loss": 2.5214290618896484, "gold_label_counts": [311, 384, 386, 355], "majority_accuracy": 0.26880222841225626}, "best_step": 0, "omega_std": 0.6023308634757996, "peak_allocated_gib": 1.11, "sample": {"gold": 2, "predicted": 3}, "elapsed_seconds": 79.1, "task": "race_middle"}
|
| 4 |
+
{"event": "packed_loss_verification", "step": 0, "passed": true, "dense_loss": 5.025767803192139, "packed_loss": 5.0257673263549805, "absolute_difference": 4.76837158203125e-07, "relative_tolerance": 0.0001, "absolute_tolerance": 1e-05, "candidates_checked": 1, "actual_training_candidates": 8, "packed_lm_tokens": 2525, "cropped_width": 432, "projection_dtype": "torch.bfloat16", "device": "cuda:0", "comparison": "Same actual training hidden states; dense masked CE versus packed valid-position CE, before the first optimizer update.", "task": "race_middle"}
|
| 5 |
+
{"event": "train", "step": 1, "total_steps": 2385, "epoch_fraction": 0.0013, "loss": 4.985269069671631, "classification_loss": 2.3983712196350098, "lm_loss": 5.173794746398926, "lr": 1.25e-05, "updates_per_second": 0.327, "peak_allocated_gib": 2.926, "nonfinite_updates": 0, "elapsed_seconds": 83.0, "task": "race_middle"}
|
| 6 |
+
{"event": "train", "step": 25, "total_steps": 2385, "epoch_fraction": 0.0315, "loss": 3.187408924102783, "classification_loss": 1.4655693769454956, "lm_loss": 3.443678855895996, "lr": 6.200105042016807e-05, "updates_per_second": 0.414, "peak_allocated_gib": 4.274, "nonfinite_updates": 0, "elapsed_seconds": 141.0, "task": "race_middle"}
|
| 7 |
+
{"event": "train", "step": 50, "total_steps": 2385, "epoch_fraction": 0.0629, "loss": 2.8200790882110596, "classification_loss": 1.2760485410690308, "lm_loss": 3.0880608558654785, "lr": 6.134453781512605e-05, "updates_per_second": 0.421, "peak_allocated_gib": 4.407, "nonfinite_updates": 0, "elapsed_seconds": 200.4, "task": "race_middle"}
|
| 8 |
+
{"event": "train", "step": 75, "total_steps": 2385, "epoch_fraction": 0.0944, "loss": 2.745144844055176, "classification_loss": 1.2592887878417969, "lm_loss": 2.971712827682495, "lr": 6.068802521008403e-05, "updates_per_second": 0.423, "peak_allocated_gib": 4.407, "nonfinite_updates": 0, "elapsed_seconds": 259.6, "task": "race_middle"}
|
| 9 |
+
{"event": "train", "step": 100, "total_steps": 2385, "epoch_fraction": 0.1259, "loss": 2.7118587493896484, "classification_loss": 1.2342157363891602, "lm_loss": 2.955287218093872, "lr": 6.003151260504202e-05, "updates_per_second": 0.419, "peak_allocated_gib": 4.407, "nonfinite_updates": 0, "elapsed_seconds": 319.3, "task": "race_middle"}
|
| 10 |
+
{"event": "train", "step": 125, "total_steps": 2385, "epoch_fraction": 0.1574, "loss": 2.675553798675537, "classification_loss": 1.2254486083984375, "lm_loss": 2.9002106189727783, "lr": 5.9375e-05, "updates_per_second": 0.412, "peak_allocated_gib": 4.407, "nonfinite_updates": 0, "elapsed_seconds": 380.0, "task": "race_middle"}
|
| 11 |
+
{"event": "train", "step": 150, "total_steps": 2385, "epoch_fraction": 0.1888, "loss": 2.6383116245269775, "classification_loss": 1.2020291090011597, "lm_loss": 2.8725645542144775, "lr": 5.871848739495799e-05, "updates_per_second": 0.406, "peak_allocated_gib": 4.618, "nonfinite_updates": 0, "elapsed_seconds": 441.5, "task": "race_middle"}
|
| 12 |
+
{"event": "train", "step": 175, "total_steps": 2385, "epoch_fraction": 0.2203, "loss": 2.6138393878936768, "classification_loss": 1.1945555210113525, "lm_loss": 2.8385674953460693, "lr": 5.806197478991597e-05, "updates_per_second": 0.423, "peak_allocated_gib": 4.618, "nonfinite_updates": 0, "elapsed_seconds": 500.7, "task": "race_middle"}
|
| 13 |
+
{"event": "train", "step": 200, "total_steps": 2385, "epoch_fraction": 0.2518, "loss": 2.585545539855957, "classification_loss": 1.1958539485931396, "lm_loss": 2.7793829441070557, "lr": 5.740546218487395e-05, "updates_per_second": 0.425, "peak_allocated_gib": 4.618, "nonfinite_updates": 0, "elapsed_seconds": 559.5, "task": "race_middle"}
|
| 14 |
+
{"event": "train", "step": 225, "total_steps": 2385, "epoch_fraction": 0.2832, "loss": 2.5741302967071533, "classification_loss": 1.1962138414382935, "lm_loss": 2.7558326721191406, "lr": 5.6748949579831934e-05, "updates_per_second": 0.422, "peak_allocated_gib": 4.618, "nonfinite_updates": 0, "elapsed_seconds": 618.8, "task": "race_middle"}
|
| 15 |
+
{"event": "train", "step": 250, "total_steps": 2385, "epoch_fraction": 0.3147, "loss": 2.5170176029205322, "classification_loss": 1.142252802848816, "lm_loss": 2.7495291233062744, "lr": 5.609243697478992e-05, "updates_per_second": 0.412, "peak_allocated_gib": 4.618, "nonfinite_updates": 0, "elapsed_seconds": 679.6, "task": "race_middle"}
|
| 16 |
+
{"event": "train", "step": 275, "total_steps": 2385, "epoch_fraction": 0.3462, "loss": 2.5560061931610107, "classification_loss": 1.192046046257019, "lm_loss": 2.727919816970825, "lr": 5.54359243697479e-05, "updates_per_second": 0.417, "peak_allocated_gib": 4.618, "nonfinite_updates": 0, "elapsed_seconds": 739.5, "task": "race_middle"}
|
| 17 |
+
{"event": "train", "step": 300, "total_steps": 2385, "epoch_fraction": 0.3776, "loss": 2.5136611461639404, "classification_loss": 1.1689919233322144, "lm_loss": 2.6893386840820312, "lr": 5.477941176470588e-05, "updates_per_second": 0.422, "peak_allocated_gib": 4.618, "nonfinite_updates": 0, "elapsed_seconds": 798.7, "task": "race_middle"}
|
| 18 |
+
{"event": "train", "step": 325, "total_steps": 2385, "epoch_fraction": 0.4091, "loss": 2.5193355083465576, "classification_loss": 1.187666654586792, "lm_loss": 2.6633379459381104, "lr": 5.412289915966387e-05, "updates_per_second": 0.42, "peak_allocated_gib": 4.618, "nonfinite_updates": 0, "elapsed_seconds": 858.2, "task": "race_middle"}
|
| 19 |
+
{"event": "train", "step": 350, "total_steps": 2385, "epoch_fraction": 0.4406, "loss": 2.499117374420166, "classification_loss": 1.1773741245269775, "lm_loss": 2.6434860229492188, "lr": 5.346638655462185e-05, "updates_per_second": 0.421, "peak_allocated_gib": 4.618, "nonfinite_updates": 0, "elapsed_seconds": 917.6, "task": "race_middle"}
|
| 20 |
+
{"event": "train", "step": 375, "total_steps": 2385, "epoch_fraction": 0.4721, "loss": 2.497126817703247, "classification_loss": 1.1829547882080078, "lm_loss": 2.6283440589904785, "lr": 5.2809873949579835e-05, "updates_per_second": 0.42, "peak_allocated_gib": 4.618, "nonfinite_updates": 0, "elapsed_seconds": 977.1, "task": "race_middle"}
|
| 21 |
+
{"event": "train", "step": 400, "total_steps": 2385, "epoch_fraction": 0.5035, "loss": 2.4380557537078857, "classification_loss": 1.1325910091400146, "lm_loss": 2.610929489135742, "lr": 5.2153361344537815e-05, "updates_per_second": 0.417, "peak_allocated_gib": 4.618, "nonfinite_updates": 0, "elapsed_seconds": 1037.0, "task": "race_middle"}
|
| 22 |
+
{"event": "eval", "step": 400, "epoch": 0, "offset": 12800, "train": {"correct": 86, "total": 128, "accuracy": 0.671875, "loss": 0.9129344820976257, "gold_label_counts": [30, 31, 38, 29], "majority_accuracy": 0.296875}, "validation": {"correct": 725, "total": 1436, "accuracy": 0.504874651810585, "loss": 1.1351072788238525, "gold_label_counts": [311, 384, 386, 355], "majority_accuracy": 0.26880222841225626}, "best_step": 400, "omega_std": 0.6023040413856506, "peak_allocated_gib": 4.618, "sample": {"gold": 2, "predicted": 2}, "elapsed_seconds": 1077.4, "task": "race_middle"}
|
| 23 |
+
{"event": "start", "step": 400, "train_examples": 25421, "epochs": 3, "total_steps": 2385, "microbatch": 2, "effective_batch": 32, "memory_cap_gib": 9.0, "trainable_parameters": 123698689, "device": "NVIDIA GeForce RTX 5060 Ti", "optimizations": {"version": 3, "training_padding": "CPU epoch plan crops each unchanged microbatch to its longest candidate, rounded up to 16 tokens (minimum 16).", "auxiliary_lm": "Project only valid next-token positions; weight each token by the inverse length of its own candidate, then average all candidates.", "device_synchronization": "One order copy to CPU per epoch; CPU-known output_size avoids synchronization when building packed CUDA indices.", "question_order_and_effective_batch": "Original seeded CUDA randperm, effective batch and microbatch membership retained.", "native_backbone": "Unchanged native model_bio.py; RACE checkpointing enabled when cropped width exceeds 384 and disabled at or below 384. SST-2 keeps its configured setting."}, "verify_packed_loss_once": false, "task": "race_middle"}
|
| 24 |
+
{"event": "train", "step": 425, "total_steps": 2385, "epoch_fraction": 0.535, "loss": 2.404723882675171, "classification_loss": 1.1139699220657349, "lm_loss": 2.581508159637451, "lr": 5.1496848739495795e-05, "updates_per_second": 0.46, "peak_allocated_gib": 6.232, "nonfinite_updates": 0, "elapsed_seconds": 1131.7, "task": "race_middle"}
|
| 25 |
+
{"event": "train", "step": 450, "total_steps": 2385, "epoch_fraction": 0.5665, "loss": 2.4390323162078857, "classification_loss": 1.1501508951187134, "lm_loss": 2.577763557434082, "lr": 5.084033613445378e-05, "updates_per_second": 0.479, "peak_allocated_gib": 6.366, "nonfinite_updates": 0, "elapsed_seconds": 1183.9, "task": "race_middle"}
|
| 26 |
+
{"event": "train", "step": 475, "total_steps": 2385, "epoch_fraction": 0.5979, "loss": 2.419196367263794, "classification_loss": 1.12021803855896, "lm_loss": 2.597956895828247, "lr": 5.018382352941177e-05, "updates_per_second": 0.461, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 1238.0, "task": "race_middle"}
|
| 27 |
+
{"event": "train", "step": 500, "total_steps": 2385, "epoch_fraction": 0.6294, "loss": 2.384754180908203, "classification_loss": 1.1209827661514282, "lm_loss": 2.5275421142578125, "lr": 4.952731092436975e-05, "updates_per_second": 0.482, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 1289.9, "task": "race_middle"}
|
| 28 |
+
{"event": "train", "step": 525, "total_steps": 2385, "epoch_fraction": 0.6609, "loss": 2.417721748352051, "classification_loss": 1.1391575336456299, "lm_loss": 2.557128429412842, "lr": 4.887079831932773e-05, "updates_per_second": 0.466, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 1343.5, "task": "race_middle"}
|
| 29 |
+
{"event": "train", "step": 550, "total_steps": 2385, "epoch_fraction": 0.6923, "loss": 2.2765934467315674, "classification_loss": 1.0210577249526978, "lm_loss": 2.51107120513916, "lr": 4.8214285714285716e-05, "updates_per_second": 0.485, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 1395.0, "task": "race_middle"}
|
| 30 |
+
{"event": "train", "step": 575, "total_steps": 2385, "epoch_fraction": 0.7238, "loss": 2.3581576347351074, "classification_loss": 1.0970054864883423, "lm_loss": 2.522304058074951, "lr": 4.7557773109243696e-05, "updates_per_second": 0.479, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 1447.2, "task": "race_middle"}
|
| 31 |
+
{"event": "train", "step": 600, "total_steps": 2385, "epoch_fraction": 0.7553, "loss": 2.3100805282592773, "classification_loss": 1.0569487810134888, "lm_loss": 2.506263256072998, "lr": 4.690126050420168e-05, "updates_per_second": 0.485, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 1498.7, "task": "race_middle"}
|
| 32 |
+
{"event": "train", "step": 625, "total_steps": 2385, "epoch_fraction": 0.7868, "loss": 2.3080027103424072, "classification_loss": 1.0624468326568604, "lm_loss": 2.4911117553710938, "lr": 4.624474789915967e-05, "updates_per_second": 0.47, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 1551.9, "task": "race_middle"}
|
| 33 |
+
{"event": "train", "step": 650, "total_steps": 2385, "epoch_fraction": 0.8182, "loss": 2.2905428409576416, "classification_loss": 1.0581409931182861, "lm_loss": 2.4648046493530273, "lr": 4.558823529411764e-05, "updates_per_second": 0.485, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 1603.4, "task": "race_middle"}
|
| 34 |
+
{"event": "train", "step": 675, "total_steps": 2385, "epoch_fraction": 0.8497, "loss": 2.301933526992798, "classification_loss": 1.0738991498947144, "lm_loss": 2.4560678005218506, "lr": 4.493172268907563e-05, "updates_per_second": 0.489, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 1654.5, "task": "race_middle"}
|
| 35 |
+
{"event": "train", "step": 700, "total_steps": 2385, "epoch_fraction": 0.8812, "loss": 2.3380091190338135, "classification_loss": 1.1155800819396973, "lm_loss": 2.4448580741882324, "lr": 4.427521008403362e-05, "updates_per_second": 0.472, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 1707.5, "task": "race_middle"}
|
| 36 |
+
{"event": "train", "step": 725, "total_steps": 2385, "epoch_fraction": 0.9126, "loss": 2.291048765182495, "classification_loss": 1.0757063627243042, "lm_loss": 2.4306845664978027, "lr": 4.36186974789916e-05, "updates_per_second": 0.475, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 1760.2, "task": "race_middle"}
|
| 37 |
+
{"event": "train", "step": 750, "total_steps": 2385, "epoch_fraction": 0.9441, "loss": 2.2462422847747803, "classification_loss": 1.040666937828064, "lm_loss": 2.4111504554748535, "lr": 4.2962184873949584e-05, "updates_per_second": 0.474, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 1812.9, "task": "race_middle"}
|
| 38 |
+
{"event": "train", "step": 775, "total_steps": 2385, "epoch_fraction": 0.9756, "loss": 2.2402682304382324, "classification_loss": 1.0352845191955566, "lm_loss": 2.4099678993225098, "lr": 4.2305672268907564e-05, "updates_per_second": 0.478, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 1865.2, "task": "race_middle"}
|
| 39 |
+
{"event": "eval", "step": 795, "epoch": 1, "offset": 0, "train": {"correct": 104, "total": 128, "accuracy": 0.8125, "loss": 0.654852032661438, "gold_label_counts": [30, 31, 38, 29], "majority_accuracy": 0.296875}, "validation": {"correct": 799, "total": 1436, "accuracy": 0.5564066852367688, "loss": 1.0419721603393555, "gold_label_counts": [311, 384, 386, 355], "majority_accuracy": 0.26880222841225626}, "best_step": 795, "omega_std": 0.6022949814796448, "peak_allocated_gib": 6.383, "sample": {"gold": 2, "predicted": 2}, "elapsed_seconds": 1945.8, "task": "race_middle"}
|
| 40 |
+
{"event": "train", "step": 800, "total_steps": 2385, "epoch_fraction": 1.0063, "loss": 2.187856435775757, "classification_loss": 1.001467227935791, "lm_loss": 2.37277889251709, "lr": 4.1649159663865544e-05, "updates_per_second": 0.265, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 1959.7, "task": "race_middle"}
|
| 41 |
+
{"event": "eval", "step": 800, "epoch": 1, "offset": 160, "train": {"correct": 105, "total": 128, "accuracy": 0.8203125, "loss": 0.5450119972229004, "gold_label_counts": [30, 31, 38, 29], "majority_accuracy": 0.296875}, "validation": {"correct": 804, "total": 1436, "accuracy": 0.5598885793871866, "loss": 1.0804342031478882, "gold_label_counts": [311, 384, 386, 355], "majority_accuracy": 0.26880222841225626}, "best_step": 800, "omega_std": 0.6022952198982239, "peak_allocated_gib": 6.383, "sample": {"gold": 2, "predicted": 2}, "elapsed_seconds": 2000.2, "task": "race_middle"}
|
| 42 |
+
{"event": "train", "step": 825, "total_steps": 2385, "epoch_fraction": 1.0378, "loss": 1.7465550899505615, "classification_loss": 0.5986884236335754, "lm_loss": 2.295733690261841, "lr": 4.099264705882353e-05, "updates_per_second": 0.486, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 2054.2, "task": "race_middle"}
|
| 43 |
+
{"event": "train", "step": 850, "total_steps": 2385, "epoch_fraction": 1.0692, "loss": 1.736619472503662, "classification_loss": 0.5899298787117004, "lm_loss": 2.293379068374634, "lr": 4.033613445378152e-05, "updates_per_second": 0.48, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 2106.4, "task": "race_middle"}
|
| 44 |
+
{"event": "train", "step": 875, "total_steps": 2385, "epoch_fraction": 1.1007, "loss": 1.6869819164276123, "classification_loss": 0.5510891079902649, "lm_loss": 2.2717854976654053, "lr": 3.967962184873949e-05, "updates_per_second": 0.477, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 2158.8, "task": "race_middle"}
|
| 45 |
+
{"event": "train", "step": 900, "total_steps": 2385, "epoch_fraction": 1.1322, "loss": 1.7243919372558594, "classification_loss": 0.5875401496887207, "lm_loss": 2.2737033367156982, "lr": 3.902310924369748e-05, "updates_per_second": 0.483, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 2210.6, "task": "race_middle"}
|
| 46 |
+
{"event": "train", "step": 925, "total_steps": 2385, "epoch_fraction": 1.1636, "loss": 1.6140456199645996, "classification_loss": 0.502417266368866, "lm_loss": 2.2232563495635986, "lr": 3.8366596638655465e-05, "updates_per_second": 0.487, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 2261.9, "task": "race_middle"}
|
| 47 |
+
{"event": "train", "step": 950, "total_steps": 2385, "epoch_fraction": 1.1951, "loss": 1.6573865413665771, "classification_loss": 0.5279656648635864, "lm_loss": 2.2588417530059814, "lr": 3.7710084033613445e-05, "updates_per_second": 0.476, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 2314.5, "task": "race_middle"}
|
| 48 |
+
{"event": "train", "step": 975, "total_steps": 2385, "epoch_fraction": 1.2266, "loss": 1.633652925491333, "classification_loss": 0.5104198455810547, "lm_loss": 2.2464661598205566, "lr": 3.705357142857143e-05, "updates_per_second": 0.484, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 2366.1, "task": "race_middle"}
|
| 49 |
+
{"event": "train", "step": 1000, "total_steps": 2385, "epoch_fraction": 1.2581, "loss": 1.6897927522659302, "classification_loss": 0.5699149966239929, "lm_loss": 2.239755630493164, "lr": 3.639705882352941e-05, "updates_per_second": 0.481, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 2418.0, "task": "race_middle"}
|
| 50 |
+
{"event": "train", "step": 1025, "total_steps": 2385, "epoch_fraction": 1.2895, "loss": 1.655150294303894, "classification_loss": 0.5479112863540649, "lm_loss": 2.214478015899658, "lr": 3.574054621848739e-05, "updates_per_second": 0.478, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 2470.4, "task": "race_middle"}
|
| 51 |
+
{"event": "train", "step": 1050, "total_steps": 2385, "epoch_fraction": 1.321, "loss": 1.676127552986145, "classification_loss": 0.5553328394889832, "lm_loss": 2.241589069366455, "lr": 3.508403361344538e-05, "updates_per_second": 0.489, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 2521.5, "task": "race_middle"}
|
| 52 |
+
{"event": "train", "step": 1075, "total_steps": 2385, "epoch_fraction": 1.3525, "loss": 1.645861268043518, "classification_loss": 0.5451458692550659, "lm_loss": 2.2014310359954834, "lr": 3.4427521008403366e-05, "updates_per_second": 0.476, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 2574.0, "task": "race_middle"}
|
| 53 |
+
{"event": "train", "step": 1100, "total_steps": 2385, "epoch_fraction": 1.3839, "loss": 1.6448230743408203, "classification_loss": 0.541966438293457, "lm_loss": 2.2057130336761475, "lr": 3.3771008403361346e-05, "updates_per_second": 0.491, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 2624.9, "task": "race_middle"}
|
| 54 |
+
{"event": "train", "step": 1125, "total_steps": 2385, "epoch_fraction": 1.4154, "loss": 1.6685564517974854, "classification_loss": 0.5655946135520935, "lm_loss": 2.205923318862915, "lr": 3.3114495798319326e-05, "updates_per_second": 0.484, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 2676.6, "task": "race_middle"}
|
| 55 |
+
{"event": "train", "step": 1150, "total_steps": 2385, "epoch_fraction": 1.4469, "loss": 1.6249042749404907, "classification_loss": 0.5286138653755188, "lm_loss": 2.1925809383392334, "lr": 3.245798319327731e-05, "updates_per_second": 0.491, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 2727.5, "task": "race_middle"}
|
| 56 |
+
{"event": "train", "step": 1175, "total_steps": 2385, "epoch_fraction": 1.4783, "loss": 1.5799367427825928, "classification_loss": 0.4795109033584595, "lm_loss": 2.2008519172668457, "lr": 3.180147058823529e-05, "updates_per_second": 0.48, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 2779.6, "task": "race_middle"}
|
| 57 |
+
{"event": "train", "step": 1200, "total_steps": 2385, "epoch_fraction": 1.5098, "loss": 1.648651123046875, "classification_loss": 0.5711127519607544, "lm_loss": 2.155076742172241, "lr": 3.114495798319328e-05, "updates_per_second": 0.494, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 2830.2, "task": "race_middle"}
|
| 58 |
+
{"event": "eval", "step": 1200, "epoch": 1, "offset": 12960, "train": {"correct": 110, "total": 128, "accuracy": 0.859375, "loss": 0.419531911611557, "gold_label_counts": [30, 31, 38, 29], "majority_accuracy": 0.296875}, "validation": {"correct": 828, "total": 1436, "accuracy": 0.5766016713091922, "loss": 1.1337844133377075, "gold_label_counts": [311, 384, 386, 355], "majority_accuracy": 0.26880222841225626}, "best_step": 1200, "omega_std": 0.6023046374320984, "peak_allocated_gib": 6.383, "sample": {"gold": 2, "predicted": 2}, "elapsed_seconds": 2869.8, "task": "race_middle"}
|
| 59 |
+
{"event": "train", "step": 1225, "total_steps": 2385, "epoch_fraction": 1.5413, "loss": 1.6209769248962402, "classification_loss": 0.5362575650215149, "lm_loss": 2.169438362121582, "lr": 3.048844537815126e-05, "updates_per_second": 0.504, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 2922.0, "task": "race_middle"}
|
| 60 |
+
{"event": "train", "step": 1250, "total_steps": 2385, "epoch_fraction": 1.5728, "loss": 1.6226683855056763, "classification_loss": 0.5399878025054932, "lm_loss": 2.165361166000366, "lr": 2.9831932773109247e-05, "updates_per_second": 0.486, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 2973.5, "task": "race_middle"}
|
| 61 |
+
{"event": "train", "step": 1275, "total_steps": 2385, "epoch_fraction": 1.6042, "loss": 1.5629373788833618, "classification_loss": 0.49103477597236633, "lm_loss": 2.143805742263794, "lr": 2.9175420168067227e-05, "updates_per_second": 0.487, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 3024.9, "task": "race_middle"}
|
| 62 |
+
{"event": "train", "step": 1300, "total_steps": 2385, "epoch_fraction": 1.6357, "loss": 1.5833594799041748, "classification_loss": 0.5137863159179688, "lm_loss": 2.139146327972412, "lr": 2.851890756302521e-05, "updates_per_second": 0.496, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 3075.3, "task": "race_middle"}
|
| 63 |
+
{"event": "train", "step": 1325, "total_steps": 2385, "epoch_fraction": 1.6672, "loss": 1.5996729135513306, "classification_loss": 0.5265106558799744, "lm_loss": 2.146324634552002, "lr": 2.7862394957983194e-05, "updates_per_second": 0.486, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 3126.7, "task": "race_middle"}
|
| 64 |
+
{"event": "train", "step": 1350, "total_steps": 2385, "epoch_fraction": 1.6986, "loss": 1.6208792924880981, "classification_loss": 0.5540426969528198, "lm_loss": 2.1336734294891357, "lr": 2.7205882352941177e-05, "updates_per_second": 0.495, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 3177.2, "task": "race_middle"}
|
| 65 |
+
{"event": "train", "step": 1375, "total_steps": 2385, "epoch_fraction": 1.7301, "loss": 1.5724105834960938, "classification_loss": 0.507769763469696, "lm_loss": 2.129281759262085, "lr": 2.654936974789916e-05, "updates_per_second": 0.498, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 3227.4, "task": "race_middle"}
|
| 66 |
+
{"event": "train", "step": 1400, "total_steps": 2385, "epoch_fraction": 1.7616, "loss": 1.5921365022659302, "classification_loss": 0.5381380915641785, "lm_loss": 2.107996702194214, "lr": 2.5892857142857144e-05, "updates_per_second": 0.492, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 3278.2, "task": "race_middle"}
|
| 67 |
+
{"event": "train", "step": 1425, "total_steps": 2385, "epoch_fraction": 1.793, "loss": 1.5692743062973022, "classification_loss": 0.5179488658905029, "lm_loss": 2.1026511192321777, "lr": 2.5236344537815128e-05, "updates_per_second": 0.483, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 3329.9, "task": "race_middle"}
|
| 68 |
+
{"event": "train", "step": 1450, "total_steps": 2385, "epoch_fraction": 1.8245, "loss": 1.5558133125305176, "classification_loss": 0.4975822865962982, "lm_loss": 2.116461992263794, "lr": 2.4579831932773108e-05, "updates_per_second": 0.487, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 3381.2, "task": "race_middle"}
|
| 69 |
+
{"event": "train", "step": 1475, "total_steps": 2385, "epoch_fraction": 1.856, "loss": 1.5773658752441406, "classification_loss": 0.5169287919998169, "lm_loss": 2.1208739280700684, "lr": 2.392331932773109e-05, "updates_per_second": 0.494, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 3431.9, "task": "race_middle"}
|
| 70 |
+
{"event": "train", "step": 1500, "total_steps": 2385, "epoch_fraction": 1.8875, "loss": 1.5817583799362183, "classification_loss": 0.5357415080070496, "lm_loss": 2.092033624649048, "lr": 2.3266806722689075e-05, "updates_per_second": 0.493, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 3482.6, "task": "race_middle"}
|
| 71 |
+
{"event": "train", "step": 1525, "total_steps": 2385, "epoch_fraction": 1.9189, "loss": 1.575385570526123, "classification_loss": 0.5256192088127136, "lm_loss": 2.0995328426361084, "lr": 2.261029411764706e-05, "updates_per_second": 0.474, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 3535.3, "task": "race_middle"}
|
| 72 |
+
{"event": "train", "step": 1550, "total_steps": 2385, "epoch_fraction": 1.9504, "loss": 1.5760983228683472, "classification_loss": 0.5207175612449646, "lm_loss": 2.110762119293213, "lr": 2.1953781512605042e-05, "updates_per_second": 0.482, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 3587.2, "task": "race_middle"}
|
| 73 |
+
{"event": "train", "step": 1575, "total_steps": 2385, "epoch_fraction": 1.9819, "loss": 1.506273627281189, "classification_loss": 0.4642983376979828, "lm_loss": 2.0839500427246094, "lr": 2.1297268907563026e-05, "updates_per_second": 0.5, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 3637.1, "task": "race_middle"}
|
| 74 |
+
{"event": "eval", "step": 1590, "epoch": 2, "offset": 0, "train": {"correct": 125, "total": 128, "accuracy": 0.9765625, "loss": 0.11218267679214478, "gold_label_counts": [30, 31, 38, 29], "majority_accuracy": 0.296875}, "validation": {"correct": 838, "total": 1436, "accuracy": 0.5835654596100278, "loss": 1.1666312217712402, "gold_label_counts": [311, 384, 386, 355], "majority_accuracy": 0.26880222841225626}, "best_step": 1590, "omega_std": 0.602307915687561, "peak_allocated_gib": 6.383, "sample": {"gold": 2, "predicted": 2}, "elapsed_seconds": 3708.1, "task": "race_middle"}
|
| 75 |
+
{"event": "train", "step": 1600, "total_steps": 2385, "epoch_fraction": 2.0126, "loss": 1.3780901432037354, "classification_loss": 0.34433525800704956, "lm_loss": 2.067509889602661, "lr": 2.064075630252101e-05, "updates_per_second": 0.263, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 3732.3, "task": "race_middle"}
|
| 76 |
+
{"event": "eval", "step": 1600, "epoch": 2, "offset": 320, "train": {"correct": 127, "total": 128, "accuracy": 0.9921875, "loss": 0.07212328910827637, "gold_label_counts": [30, 31, 38, 29], "majority_accuracy": 0.296875}, "validation": {"correct": 833, "total": 1436, "accuracy": 0.58008356545961, "loss": 1.5192731618881226, "gold_label_counts": [311, 384, 386, 355], "majority_accuracy": 0.26880222841225626}, "best_step": 1590, "omega_std": 0.6023086905479431, "peak_allocated_gib": 6.383, "sample": {"gold": 2, "predicted": 2}, "elapsed_seconds": 3772.9, "task": "race_middle"}
|
| 77 |
+
{"event": "train", "step": 1625, "total_steps": 2385, "epoch_fraction": 2.0441, "loss": 1.0977911949157715, "classification_loss": 0.09246373921632767, "lm_loss": 2.010654926300049, "lr": 1.998424369747899e-05, "updates_per_second": 0.483, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 3825.9, "task": "race_middle"}
|
| 78 |
+
{"event": "train", "step": 1650, "total_steps": 2385, "epoch_fraction": 2.0755, "loss": 1.1537387371063232, "classification_loss": 0.14970183372497559, "lm_loss": 2.0080740451812744, "lr": 1.9327731092436976e-05, "updates_per_second": 0.47, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 3879.1, "task": "race_middle"}
|
| 79 |
+
{"event": "train", "step": 1675, "total_steps": 2385, "epoch_fraction": 2.107, "loss": 1.1092095375061035, "classification_loss": 0.11812342703342438, "lm_loss": 1.9821720123291016, "lr": 1.867121848739496e-05, "updates_per_second": 0.474, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 3931.8, "task": "race_middle"}
|
| 80 |
+
{"event": "train", "step": 1700, "total_steps": 2385, "epoch_fraction": 2.1385, "loss": 1.1044018268585205, "classification_loss": 0.0959235429763794, "lm_loss": 2.0169565677642822, "lr": 1.801470588235294e-05, "updates_per_second": 0.466, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 3985.5, "task": "race_middle"}
|
| 81 |
+
{"event": "train", "step": 1725, "total_steps": 2385, "epoch_fraction": 2.1699, "loss": 1.086236596107483, "classification_loss": 0.10009206831455231, "lm_loss": 1.9722895622253418, "lr": 1.7358193277310926e-05, "updates_per_second": 0.479, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 4037.7, "task": "race_middle"}
|
| 82 |
+
{"event": "train", "step": 1750, "total_steps": 2385, "epoch_fraction": 2.2014, "loss": 1.0760489702224731, "classification_loss": 0.09492538124322891, "lm_loss": 1.9622471332550049, "lr": 1.6701680672268907e-05, "updates_per_second": 0.487, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 4089.0, "task": "race_middle"}
|
| 83 |
+
{"event": "train", "step": 1775, "total_steps": 2385, "epoch_fraction": 2.2329, "loss": 1.131264328956604, "classification_loss": 0.1423715502023697, "lm_loss": 1.977785348892212, "lr": 1.604516806722689e-05, "updates_per_second": 0.473, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 4141.9, "task": "race_middle"}
|
| 84 |
+
{"event": "train", "step": 1800, "total_steps": 2385, "epoch_fraction": 2.2643, "loss": 1.1268174648284912, "classification_loss": 0.1479249745607376, "lm_loss": 1.9577851295471191, "lr": 1.5388655462184874e-05, "updates_per_second": 0.49, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 4192.9, "task": "race_middle"}
|
| 85 |
+
{"event": "train", "step": 1825, "total_steps": 2385, "epoch_fraction": 2.2958, "loss": 1.058220386505127, "classification_loss": 0.0932118222117424, "lm_loss": 1.9300169944763184, "lr": 1.4732142857142857e-05, "updates_per_second": 0.482, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 4244.8, "task": "race_middle"}
|
| 86 |
+
{"event": "train", "step": 1850, "total_steps": 2385, "epoch_fraction": 2.3273, "loss": 1.103035807609558, "classification_loss": 0.12815970182418823, "lm_loss": 1.9497523307800293, "lr": 1.407563025210084e-05, "updates_per_second": 0.472, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 4297.8, "task": "race_middle"}
|
| 87 |
+
{"event": "train", "step": 1875, "total_steps": 2385, "epoch_fraction": 2.3588, "loss": 1.0581059455871582, "classification_loss": 0.09285680204629898, "lm_loss": 1.930498480796814, "lr": 1.3419117647058822e-05, "updates_per_second": 0.467, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 4351.3, "task": "race_middle"}
|
| 88 |
+
{"event": "train", "step": 1900, "total_steps": 2385, "epoch_fraction": 2.3902, "loss": 1.0412935018539429, "classification_loss": 0.09380818158388138, "lm_loss": 1.8949710130691528, "lr": 1.2762605042016808e-05, "updates_per_second": 0.49, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 4402.3, "task": "race_middle"}
|
| 89 |
+
{"event": "train", "step": 1925, "total_steps": 2385, "epoch_fraction": 2.4217, "loss": 1.0839320421218872, "classification_loss": 0.12270724773406982, "lm_loss": 1.9224497079849243, "lr": 1.2106092436974791e-05, "updates_per_second": 0.466, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 4456.0, "task": "race_middle"}
|
| 90 |
+
{"event": "train", "step": 1950, "total_steps": 2385, "epoch_fraction": 2.4532, "loss": 1.0358823537826538, "classification_loss": 0.09550299495458603, "lm_loss": 1.8807587623596191, "lr": 1.1449579831932773e-05, "updates_per_second": 0.479, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 4508.2, "task": "race_middle"}
|
| 91 |
+
{"event": "train", "step": 1975, "total_steps": 2385, "epoch_fraction": 2.4846, "loss": 1.0827490091323853, "classification_loss": 0.1339101642370224, "lm_loss": 1.8976777791976929, "lr": 1.0793067226890756e-05, "updates_per_second": 0.489, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 4559.3, "task": "race_middle"}
|
| 92 |
+
{"event": "train", "step": 2000, "total_steps": 2385, "epoch_fraction": 2.5161, "loss": 1.06181800365448, "classification_loss": 0.10987097769975662, "lm_loss": 1.903894305229187, "lr": 1.013655462184874e-05, "updates_per_second": 0.471, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 4612.4, "task": "race_middle"}
|
| 93 |
+
{"event": "eval", "step": 2000, "epoch": 2, "offset": 13120, "train": {"correct": 126, "total": 128, "accuracy": 0.984375, "loss": 0.03775814548134804, "gold_label_counts": [30, 31, 38, 29], "majority_accuracy": 0.296875}, "validation": {"correct": 843, "total": 1436, "accuracy": 0.5870473537604457, "loss": 2.259791851043701, "gold_label_counts": [311, 384, 386, 355], "majority_accuracy": 0.26880222841225626}, "best_step": 2000, "omega_std": 0.6023051738739014, "peak_allocated_gib": 6.383, "sample": {"gold": 2, "predicted": 2}, "elapsed_seconds": 4653.2, "task": "race_middle"}
|
| 94 |
+
{"event": "train", "step": 2025, "total_steps": 2385, "epoch_fraction": 2.5476, "loss": 1.0426304340362549, "classification_loss": 0.12839677929878235, "lm_loss": 1.8284672498703003, "lr": 9.480042016806722e-06, "updates_per_second": 0.492, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 4706.6, "task": "race_middle"}
|
| 95 |
+
{"event": "train", "step": 2050, "total_steps": 2385, "epoch_fraction": 2.579, "loss": 1.0208618640899658, "classification_loss": 0.08157386630773544, "lm_loss": 1.8785756826400757, "lr": 8.823529411764705e-06, "updates_per_second": 0.483, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 4758.4, "task": "race_middle"}
|
| 96 |
+
{"event": "train", "step": 2075, "total_steps": 2385, "epoch_fraction": 2.6105, "loss": 1.030715823173523, "classification_loss": 0.09123934805393219, "lm_loss": 1.8789527416229248, "lr": 8.16701680672269e-06, "updates_per_second": 0.471, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 4811.4, "task": "race_middle"}
|
| 97 |
+
{"event": "train", "step": 2100, "total_steps": 2385, "epoch_fraction": 2.642, "loss": 1.0435203313827515, "classification_loss": 0.09825462847948074, "lm_loss": 1.8905309438705444, "lr": 7.510504201680673e-06, "updates_per_second": 0.469, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 4864.8, "task": "race_middle"}
|
| 98 |
+
{"event": "train", "step": 2125, "total_steps": 2385, "epoch_fraction": 2.6735, "loss": 1.0152134895324707, "classification_loss": 0.0888785645365715, "lm_loss": 1.8526700735092163, "lr": 6.853991596638656e-06, "updates_per_second": 0.476, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 4917.3, "task": "race_middle"}
|
| 99 |
+
{"event": "train", "step": 2150, "total_steps": 2385, "epoch_fraction": 2.7049, "loss": 1.0472594499588013, "classification_loss": 0.10265874862670898, "lm_loss": 1.8892017602920532, "lr": 6.197478991596638e-06, "updates_per_second": 0.464, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 4971.2, "task": "race_middle"}
|
| 100 |
+
{"event": "train", "step": 2175, "total_steps": 2385, "epoch_fraction": 2.7364, "loss": 1.04432213306427, "classification_loss": 0.11284386366605759, "lm_loss": 1.8629567623138428, "lr": 5.5409663865546226e-06, "updates_per_second": 0.489, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 5022.3, "task": "race_middle"}
|
| 101 |
+
{"event": "train", "step": 2200, "total_steps": 2385, "epoch_fraction": 2.7679, "loss": 1.007009506225586, "classification_loss": 0.09896551817655563, "lm_loss": 1.816088318824768, "lr": 4.884453781512605e-06, "updates_per_second": 0.494, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 5072.9, "task": "race_middle"}
|
| 102 |
+
{"event": "train", "step": 2225, "total_steps": 2385, "epoch_fraction": 2.7993, "loss": 1.0223346948623657, "classification_loss": 0.1071593165397644, "lm_loss": 1.8303507566452026, "lr": 4.227941176470588e-06, "updates_per_second": 0.484, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 5124.6, "task": "race_middle"}
|
| 103 |
+
{"event": "train", "step": 2250, "total_steps": 2385, "epoch_fraction": 2.8308, "loss": 1.0083647966384888, "classification_loss": 0.08779270946979523, "lm_loss": 1.8411445617675781, "lr": 3.5714285714285714e-06, "updates_per_second": 0.482, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 5176.5, "task": "race_middle"}
|
| 104 |
+
{"event": "train", "step": 2275, "total_steps": 2385, "epoch_fraction": 2.8623, "loss": 0.9689734578132629, "classification_loss": 0.05670125409960747, "lm_loss": 1.8245441913604736, "lr": 2.914915966386555e-06, "updates_per_second": 0.483, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 5228.3, "task": "race_middle"}
|
| 105 |
+
{"event": "train", "step": 2300, "total_steps": 2385, "epoch_fraction": 2.8937, "loss": 0.9841031432151794, "classification_loss": 0.06860894709825516, "lm_loss": 1.8309884071350098, "lr": 2.258403361344538e-06, "updates_per_second": 0.48, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 5280.3, "task": "race_middle"}
|
| 106 |
+
{"event": "train", "step": 2325, "total_steps": 2385, "epoch_fraction": 2.9252, "loss": 0.9719945788383484, "classification_loss": 0.06914593279361725, "lm_loss": 1.8056970834732056, "lr": 1.601890756302521e-06, "updates_per_second": 0.476, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 5332.9, "task": "race_middle"}
|
| 107 |
+
{"event": "train", "step": 2350, "total_steps": 2385, "epoch_fraction": 2.9567, "loss": 1.0059738159179688, "classification_loss": 0.0832299292087555, "lm_loss": 1.8454878330230713, "lr": 9.453781512605043e-07, "updates_per_second": 0.477, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 5385.2, "task": "race_middle"}
|
| 108 |
+
{"event": "train", "step": 2375, "total_steps": 2385, "epoch_fraction": 2.9882, "loss": 1.0165380239486694, "classification_loss": 0.10338052362203598, "lm_loss": 1.826314926147461, "lr": 2.888655462184874e-07, "updates_per_second": 0.466, "peak_allocated_gib": 6.383, "nonfinite_updates": 0, "elapsed_seconds": 5438.8, "task": "race_middle"}
|
| 109 |
+
{"event": "eval", "step": 2385, "epoch": 3, "offset": 0, "train": {"correct": 128, "total": 128, "accuracy": 1.0, "loss": 0.014566410332918167, "gold_label_counts": [30, 31, 38, 29], "majority_accuracy": 0.296875}, "validation": {"correct": 850, "total": 1436, "accuracy": 0.5919220055710307, "loss": 2.641692876815796, "gold_label_counts": [311, 384, 386, 355], "majority_accuracy": 0.26880222841225626}, "best_step": 2385, "omega_std": 0.6023051738739014, "peak_allocated_gib": 6.383, "sample": {"gold": 2, "predicted": 2}, "elapsed_seconds": 5499.9, "task": "race_middle"}
|
| 110 |
+
{"event": "complete", "step": 2385, "best_step": 2385, "stopping_cause": "epochs_complete", "heldout": "test", "before": {"correct": 385, "total": 1436, "accuracy": 0.2681058495821727, "loss": 2.6538400650024414, "gold_label_counts": [309, 388, 374, 365], "majority_accuracy": 0.27019498607242337}, "after": {"correct": 854, "total": 1436, "accuracy": 0.5947075208913649, "loss": 2.4820992946624756, "gold_label_counts": [309, 388, 374, 365], "majority_accuracy": 0.27019498607242337}, "paired_probe": {"before_correct": 11, "after_correct": 27, "total": 40}, "elapsed_seconds": 5540.6, "task": "race_middle"}
|
task_finetuning/results/race_middle_provenance.json
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_source_sha256": "a28201b8194941a75d3ca2e0812e6d2d0eef1b2656cc0c9f8fd0ca65bf96b28b",
|
| 3 |
+
"trainer_sha256": "aaa80085a75147c02945e7e09bcdd0d18c929a79a12ffce2a830fa29c5d81cfe",
|
| 4 |
+
"starting_checkpoint": {
|
| 5 |
+
"checkpoint": "original_workspace/Main folders/WiggleGPT v3/WiggleGPT/out-wigglegpt-pure-124m/ckpt.pt",
|
| 6 |
+
"saved_step": 600000,
|
| 7 |
+
"parameters": 123697920,
|
| 8 |
+
"model_args": {
|
| 9 |
+
"n_layer": 12,
|
| 10 |
+
"n_head": 12,
|
| 11 |
+
"n_embd": 768,
|
| 12 |
+
"block_size": 1024,
|
| 13 |
+
"bias": false,
|
| 14 |
+
"vocab_size": 50304,
|
| 15 |
+
"dropout": 0.0,
|
| 16 |
+
"use_bio_mlp": true,
|
| 17 |
+
"use_rmsnorm": true,
|
| 18 |
+
"use_rope": true,
|
| 19 |
+
"gradient_checkpointing": false
|
| 20 |
+
},
|
| 21 |
+
"torch": "2.11.0.dev20260215+cu128",
|
| 22 |
+
"checkpoint_sha256": "88afaab6c93ef3e33940b4e25cbce2f44038c428109600afe8285586a3a767b9"
|
| 23 |
+
},
|
| 24 |
+
"torch": "2.11.0.dev20260215+cu128",
|
| 25 |
+
"markers": {
|
| 26 |
+
"start": 50257,
|
| 27 |
+
"delimiter": 50258,
|
| 28 |
+
"classify": 50259,
|
| 29 |
+
"pad": 50256
|
| 30 |
+
},
|
| 31 |
+
"native_backbone_dropout": 0.0,
|
| 32 |
+
"data": {
|
| 33 |
+
"train": {
|
| 34 |
+
"example_ids_summary": {
|
| 35 |
+
"count": 25421,
|
| 36 |
+
"sha256_json": "d7830571364e17c9f20533af95cb0a01cb686fef1467f91eaa065d563e8d7abd"
|
| 37 |
+
},
|
| 38 |
+
"metadata": {
|
| 39 |
+
"task": "race_middle",
|
| 40 |
+
"split": "train",
|
| 41 |
+
"examples": 25421,
|
| 42 |
+
"choices": 4,
|
| 43 |
+
"source_jsonl": "original_workspace/output/gpt1_style/task_finetuning/data/race_middle_train.jsonl",
|
| 44 |
+
"source_sha256": "ed805e44773ff5856211f0713270222c8ca681e84c43b549cdecfbef97a46652",
|
| 45 |
+
"pad_length": 512,
|
| 46 |
+
"max_sequence_length": 512,
|
| 47 |
+
"min_sequence_length": 20,
|
| 48 |
+
"max_untruncated_length": 673,
|
| 49 |
+
"truncation": {
|
| 50 |
+
"examples_with_truncation": 210,
|
| 51 |
+
"candidate_sequences_with_truncation": 840,
|
| 52 |
+
"total_tokens_removed": 10052,
|
| 53 |
+
"max_tokens_removed_per_example": 161
|
| 54 |
+
},
|
| 55 |
+
"context_policy": "[START] + shared article prefix + [DELIM] + complete question + [DELIM] + complete answer + [CLS]; all four choices retain the same article prefix, budgeted using the longest answer; right-pad to 512.",
|
| 56 |
+
"input_ids_dtype": "int32",
|
| 57 |
+
"lengths_dtype": "int64",
|
| 58 |
+
"labels_dtype": "int64"
|
| 59 |
+
},
|
| 60 |
+
"source_sha256": "ed805e44773ff5856211f0713270222c8ca681e84c43b549cdecfbef97a46652"
|
| 61 |
+
},
|
| 62 |
+
"validation": {
|
| 63 |
+
"example_ids_summary": {
|
| 64 |
+
"count": 1436,
|
| 65 |
+
"sha256_json": "6cf3bb9048136b26ef8bee6be1ce9c9245e35d7522f93817b3d226be26cf9a76"
|
| 66 |
+
},
|
| 67 |
+
"metadata": {
|
| 68 |
+
"task": "race_middle",
|
| 69 |
+
"split": "validation",
|
| 70 |
+
"examples": 1436,
|
| 71 |
+
"choices": 4,
|
| 72 |
+
"source_jsonl": "original_workspace/output/gpt1_style/task_finetuning/data/race_middle_validation.jsonl",
|
| 73 |
+
"source_sha256": "385d417aaddbc25038d7bd4afbf9f385d4e3ae74bff532397ba938b0fbf0f622",
|
| 74 |
+
"pad_length": 512,
|
| 75 |
+
"max_sequence_length": 512,
|
| 76 |
+
"min_sequence_length": 65,
|
| 77 |
+
"max_untruncated_length": 530,
|
| 78 |
+
"truncation": {
|
| 79 |
+
"examples_with_truncation": 9,
|
| 80 |
+
"candidate_sequences_with_truncation": 36,
|
| 81 |
+
"total_tokens_removed": 71,
|
| 82 |
+
"max_tokens_removed_per_example": 18
|
| 83 |
+
},
|
| 84 |
+
"context_policy": "[START] + shared article prefix + [DELIM] + complete question + [DELIM] + complete answer + [CLS]; all four choices retain the same article prefix, budgeted using the longest answer; right-pad to 512.",
|
| 85 |
+
"input_ids_dtype": "int32",
|
| 86 |
+
"lengths_dtype": "int64",
|
| 87 |
+
"labels_dtype": "int64"
|
| 88 |
+
},
|
| 89 |
+
"source_sha256": "385d417aaddbc25038d7bd4afbf9f385d4e3ae74bff532397ba938b0fbf0f622"
|
| 90 |
+
},
|
| 91 |
+
"test": {
|
| 92 |
+
"example_ids_summary": {
|
| 93 |
+
"count": 1436,
|
| 94 |
+
"sha256_json": "09b875e7553744bf6ebc8334df4277de389fab670a84e1a131a76ed2fac5253e"
|
| 95 |
+
},
|
| 96 |
+
"metadata": {
|
| 97 |
+
"task": "race_middle",
|
| 98 |
+
"split": "test",
|
| 99 |
+
"examples": 1436,
|
| 100 |
+
"choices": 4,
|
| 101 |
+
"source_jsonl": "original_workspace/output/gpt1_style/task_finetuning/data/race_middle_test.jsonl",
|
| 102 |
+
"source_sha256": "dd9756cf41d600260b59064c2b10395affb53d769a4bf71017ca8dc6ceb005d9",
|
| 103 |
+
"pad_length": 512,
|
| 104 |
+
"max_sequence_length": 512,
|
| 105 |
+
"min_sequence_length": 41,
|
| 106 |
+
"max_untruncated_length": 565,
|
| 107 |
+
"truncation": {
|
| 108 |
+
"examples_with_truncation": 14,
|
| 109 |
+
"candidate_sequences_with_truncation": 56,
|
| 110 |
+
"total_tokens_removed": 531,
|
| 111 |
+
"max_tokens_removed_per_example": 53
|
| 112 |
+
},
|
| 113 |
+
"context_policy": "[START] + shared article prefix + [DELIM] + complete question + [DELIM] + complete answer + [CLS]; all four choices retain the same article prefix, budgeted using the longest answer; right-pad to 512.",
|
| 114 |
+
"input_ids_dtype": "int32",
|
| 115 |
+
"lengths_dtype": "int64",
|
| 116 |
+
"labels_dtype": "int64"
|
| 117 |
+
},
|
| 118 |
+
"source_sha256": "dd9756cf41d600260b59064c2b10395affb53d769a4bf71017ca8dc6ceb005d9"
|
| 119 |
+
}
|
| 120 |
+
},
|
| 121 |
+
"all_backbone_parameters_train": true,
|
| 122 |
+
"optimizations": {
|
| 123 |
+
"version": 2,
|
| 124 |
+
"training_padding": "CPU epoch plan crops each unchanged microbatch to its longest candidate, rounded up to 16 tokens (minimum 16).",
|
| 125 |
+
"auxiliary_lm": "Project only valid next-token positions; weight each token by the inverse length of its own candidate, then average all candidates.",
|
| 126 |
+
"device_synchronization": "One order copy to CPU per epoch; CPU-known output_size avoids synchronization when building packed CUDA indices.",
|
| 127 |
+
"question_order_and_effective_batch": "Original seeded CUDA randperm, effective batch and microbatch membership retained.",
|
| 128 |
+
"native_backbone": "Unchanged native model_bio.py and configured gradient checkpointing retained."
|
| 129 |
+
}
|
| 130 |
+
}
|
task_finetuning/results/race_middle_resume_provenance.json
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"resume_checkpoint": "original_workspace/output/gpt1_style/task_finetuning/runs/train_tasks_v2_2026-10-26_06-16-49_race_middle/latest.pt",
|
| 3 |
+
"resume_checkpoint_sha256": "abf2cbcec103ff914738edcf57a218252fd184d90bca403bd1d1c25732a03528",
|
| 4 |
+
"start_step": 400,
|
| 5 |
+
"epoch": 0,
|
| 6 |
+
"offset": 12800,
|
| 7 |
+
"trainer_source": "train_tasks_v3.py",
|
| 8 |
+
"trainer_sha256": "080b06025459b59c2dddb2be046f28c5906e40915dbbb325a8c64642511d5517",
|
| 9 |
+
"model_source_sha256": "a28201b8194941a75d3ca2e0812e6d2d0eef1b2656cc0c9f8fd0ca65bf96b28b",
|
| 10 |
+
"configuration_source": "Saved resume checkpoint configuration",
|
| 11 |
+
"optimizations": {
|
| 12 |
+
"version": 3,
|
| 13 |
+
"training_padding": "CPU epoch plan crops each unchanged microbatch to its longest candidate, rounded up to 16 tokens (minimum 16).",
|
| 14 |
+
"auxiliary_lm": "Project only valid next-token positions; weight each token by the inverse length of its own candidate, then average all candidates.",
|
| 15 |
+
"device_synchronization": "One order copy to CPU per epoch; CPU-known output_size avoids synchronization when building packed CUDA indices.",
|
| 16 |
+
"question_order_and_effective_batch": "Original seeded CUDA randperm, effective batch and microbatch membership retained.",
|
| 17 |
+
"native_backbone": "Unchanged native model_bio.py; RACE checkpointing enabled when cropped width exceeds 384 and disabled at or below 384. SST-2 keeps its configured setting."
|
| 18 |
+
}
|
| 19 |
+
}
|
task_finetuning/results/sst2_metrics.jsonl
ADDED
|
@@ -0,0 +1,273 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"event": "start", "step": 0, "train_examples": 67349, "epochs": 3, "total_steps": 6315, "microbatch": 16, "effective_batch": 32, "memory_cap_gib": 9.0, "trainable_parameters": 123699458, "device": "NVIDIA GeForce RTX 5060 Ti", "task": "sst2"}
|
| 2 |
+
{"event": "before", "heldout": "validation", "correct": 437, "total": 872, "accuracy": 0.5011467889908257, "loss": 4.14792537689209, "gold_label_counts": [428, 444], "majority_accuracy": 0.5091743119266054, "task": "sst2"}
|
| 3 |
+
{"event": "eval", "step": 0, "epoch": 0, "offset": 0, "train": {"correct": 70, "total": 128, "accuracy": 0.546875, "loss": 2.8148393630981445, "gold_label_counts": [65, 63], "majority_accuracy": 0.5078125}, "validation": {"correct": 437, "total": 872, "accuracy": 0.5011467889908257, "loss": 4.14792537689209, "gold_label_counts": [428, 444], "majority_accuracy": 0.5091743119266054}, "best_step": 0, "omega_std": 0.6023308634757996, "peak_allocated_gib": 0.606, "sample": {"gold": 1, "predicted": 0}, "elapsed_seconds": 3.9, "task": "sst2"}
|
| 4 |
+
{"event": "train", "step": 1, "total_steps": 6315, "epoch_fraction": 0.0005, "loss": 9.488792419433594, "classification_loss": 3.114635705947876, "lm_loss": 12.74831485748291, "lr": 4.807692307692308e-06, "updates_per_second": 1.697, "peak_allocated_gib": 2.91, "nonfinite_updates": 0, "elapsed_seconds": 5.4, "task": "sst2"}
|
| 5 |
+
{"event": "train", "step": 25, "total_steps": 6315, "epoch_fraction": 0.0119, "loss": 5.967916011810303, "classification_loss": 2.947064161300659, "lm_loss": 6.041703701019287, "lr": 6.23909076483656e-05, "updates_per_second": 6.339, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 9.2, "task": "sst2"}
|
| 6 |
+
{"event": "train", "step": 50, "total_steps": 6315, "epoch_fraction": 0.0238, "loss": 4.922191143035889, "classification_loss": 2.607499837875366, "lm_loss": 4.629383563995361, "lr": 6.214297048556014e-05, "updates_per_second": 6.359, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 13.1, "task": "sst2"}
|
| 7 |
+
{"event": "train", "step": 75, "total_steps": 6315, "epoch_fraction": 0.0356, "loss": 3.5110924243927, "classification_loss": 1.3375811576843262, "lm_loss": 4.34702205657959, "lr": 6.189503332275467e-05, "updates_per_second": 6.373, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 17.0, "task": "sst2"}
|
| 8 |
+
{"event": "train", "step": 100, "total_steps": 6315, "epoch_fraction": 0.0475, "loss": 2.9477453231811523, "classification_loss": 0.7837342023849487, "lm_loss": 4.328023433685303, "lr": 6.164709615994923e-05, "updates_per_second": 6.399, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 20.9, "task": "sst2"}
|
| 9 |
+
{"event": "train", "step": 125, "total_steps": 6315, "epoch_fraction": 0.0594, "loss": 2.901278257369995, "classification_loss": 0.7659029960632324, "lm_loss": 4.270751476287842, "lr": 6.139915899714377e-05, "updates_per_second": 6.42, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 24.8, "task": "sst2"}
|
| 10 |
+
{"event": "train", "step": 150, "total_steps": 6315, "epoch_fraction": 0.0713, "loss": 3.120953321456909, "classification_loss": 1.0203531980514526, "lm_loss": 4.201200008392334, "lr": 6.11512218343383e-05, "updates_per_second": 6.391, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 28.7, "task": "sst2"}
|
| 11 |
+
{"event": "train", "step": 175, "total_steps": 6315, "epoch_fraction": 0.0831, "loss": 2.7226736545562744, "classification_loss": 0.6115703582763672, "lm_loss": 4.222206115722656, "lr": 6.090328467153284e-05, "updates_per_second": 6.397, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 32.6, "task": "sst2"}
|
| 12 |
+
{"event": "train", "step": 200, "total_steps": 6315, "epoch_fraction": 0.095, "loss": 2.7715611457824707, "classification_loss": 0.6940281391143799, "lm_loss": 4.15506649017334, "lr": 6.065534750872739e-05, "updates_per_second": 6.426, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 36.5, "task": "sst2"}
|
| 13 |
+
{"event": "train", "step": 225, "total_steps": 6315, "epoch_fraction": 0.1069, "loss": 2.6003851890563965, "classification_loss": 0.5685897469520569, "lm_loss": 4.063590049743652, "lr": 6.0407410345921934e-05, "updates_per_second": 6.424, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 40.4, "task": "sst2"}
|
| 14 |
+
{"event": "train", "step": 250, "total_steps": 6315, "epoch_fraction": 0.1188, "loss": 2.8825366497039795, "classification_loss": 0.8409788608551025, "lm_loss": 4.083115577697754, "lr": 6.015947318311647e-05, "updates_per_second": 6.43, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 44.3, "task": "sst2"}
|
| 15 |
+
{"event": "train", "step": 275, "total_steps": 6315, "epoch_fraction": 0.1307, "loss": 3.0506155490875244, "classification_loss": 1.043200969696045, "lm_loss": 4.014829158782959, "lr": 5.991153602031101e-05, "updates_per_second": 6.422, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 48.2, "task": "sst2"}
|
| 16 |
+
{"event": "train", "step": 300, "total_steps": 6315, "epoch_fraction": 0.1425, "loss": 2.511808395385742, "classification_loss": 0.49926748871803284, "lm_loss": 4.025081634521484, "lr": 5.966359885750555e-05, "updates_per_second": 6.432, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 52.1, "task": "sst2"}
|
| 17 |
+
{"event": "train", "step": 325, "total_steps": 6315, "epoch_fraction": 0.1544, "loss": 2.4246182441711426, "classification_loss": 0.43881192803382874, "lm_loss": 3.9716126918792725, "lr": 5.94156616947001e-05, "updates_per_second": 6.407, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 56.0, "task": "sst2"}
|
| 18 |
+
{"event": "train", "step": 350, "total_steps": 6315, "epoch_fraction": 0.1663, "loss": 2.6543045043945312, "classification_loss": 0.6687469482421875, "lm_loss": 3.971115827560425, "lr": 5.9167724531894636e-05, "updates_per_second": 6.372, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 59.9, "task": "sst2"}
|
| 19 |
+
{"event": "train", "step": 375, "total_steps": 6315, "epoch_fraction": 0.1782, "loss": 2.3760600090026855, "classification_loss": 0.408420592546463, "lm_loss": 3.9352781772613525, "lr": 5.891978736908918e-05, "updates_per_second": 6.399, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 63.8, "task": "sst2"}
|
| 20 |
+
{"event": "train", "step": 400, "total_steps": 6315, "epoch_fraction": 0.1901, "loss": 2.451561450958252, "classification_loss": 0.5121500492095947, "lm_loss": 3.8788225650787354, "lr": 5.867185020628372e-05, "updates_per_second": 6.407, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 67.7, "task": "sst2"}
|
| 21 |
+
{"event": "train", "step": 425, "total_steps": 6315, "epoch_fraction": 0.2019, "loss": 2.7101783752441406, "classification_loss": 0.7416963577270508, "lm_loss": 3.9369633197784424, "lr": 5.8423913043478266e-05, "updates_per_second": 6.366, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 71.7, "task": "sst2"}
|
| 22 |
+
{"event": "train", "step": 450, "total_steps": 6315, "epoch_fraction": 0.2138, "loss": 2.344588041305542, "classification_loss": 0.43529579043388367, "lm_loss": 3.818584442138672, "lr": 5.8175975880672805e-05, "updates_per_second": 6.507, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 75.5, "task": "sst2"}
|
| 23 |
+
{"event": "train", "step": 475, "total_steps": 6315, "epoch_fraction": 0.2257, "loss": 2.34840989112854, "classification_loss": 0.41136229038238525, "lm_loss": 3.8740947246551514, "lr": 5.7928038717867344e-05, "updates_per_second": 6.462, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 79.4, "task": "sst2"}
|
| 24 |
+
{"event": "train", "step": 500, "total_steps": 6315, "epoch_fraction": 0.2376, "loss": 2.2097387313842773, "classification_loss": 0.3122953772544861, "lm_loss": 3.794886350631714, "lr": 5.768010155506188e-05, "updates_per_second": 6.459, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 83.2, "task": "sst2"}
|
| 25 |
+
{"event": "train", "step": 525, "total_steps": 6315, "epoch_fraction": 0.2494, "loss": 2.2180964946746826, "classification_loss": 0.3469282388687134, "lm_loss": 3.7423369884490967, "lr": 5.743216439225643e-05, "updates_per_second": 4.874, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 88.4, "task": "sst2"}
|
| 26 |
+
{"event": "train", "step": 550, "total_steps": 6315, "epoch_fraction": 0.2613, "loss": 2.277540445327759, "classification_loss": 0.3630802035331726, "lm_loss": 3.8289196491241455, "lr": 5.718422722945097e-05, "updates_per_second": 6.511, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 92.2, "task": "sst2"}
|
| 27 |
+
{"event": "train", "step": 575, "total_steps": 6315, "epoch_fraction": 0.2732, "loss": 2.26564359664917, "classification_loss": 0.37143492698669434, "lm_loss": 3.7884175777435303, "lr": 5.693629006664551e-05, "updates_per_second": 6.526, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 96.0, "task": "sst2"}
|
| 28 |
+
{"event": "train", "step": 600, "total_steps": 6315, "epoch_fraction": 0.2851, "loss": 2.210198402404785, "classification_loss": 0.38525518774986267, "lm_loss": 3.6498863697052, "lr": 5.668835290384005e-05, "updates_per_second": 6.109, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 100.1, "task": "sst2"}
|
| 29 |
+
{"event": "train", "step": 625, "total_steps": 6315, "epoch_fraction": 0.297, "loss": 2.2624258995056152, "classification_loss": 0.41674166917800903, "lm_loss": 3.6913676261901855, "lr": 5.64404157410346e-05, "updates_per_second": 6.293, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 104.1, "task": "sst2"}
|
| 30 |
+
{"event": "train", "step": 650, "total_steps": 6315, "epoch_fraction": 0.3088, "loss": 2.195566177368164, "classification_loss": 0.34315669536590576, "lm_loss": 3.7048187255859375, "lr": 5.6192478578229136e-05, "updates_per_second": 6.312, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 108.1, "task": "sst2"}
|
| 31 |
+
{"event": "train", "step": 675, "total_steps": 6315, "epoch_fraction": 0.3207, "loss": 2.138474464416504, "classification_loss": 0.3079281449317932, "lm_loss": 3.661092519760132, "lr": 5.5944541415423675e-05, "updates_per_second": 6.345, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 112.0, "task": "sst2"}
|
| 32 |
+
{"event": "train", "step": 700, "total_steps": 6315, "epoch_fraction": 0.3326, "loss": 2.285196542739868, "classification_loss": 0.4572332203388214, "lm_loss": 3.655926465988159, "lr": 5.5696604252618214e-05, "updates_per_second": 6.4, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 115.9, "task": "sst2"}
|
| 33 |
+
{"event": "train", "step": 725, "total_steps": 6315, "epoch_fraction": 0.3445, "loss": 2.0899930000305176, "classification_loss": 0.30079689621925354, "lm_loss": 3.578392267227173, "lr": 5.544866708981275e-05, "updates_per_second": 6.255, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 119.9, "task": "sst2"}
|
| 34 |
+
{"event": "train", "step": 750, "total_steps": 6315, "epoch_fraction": 0.3564, "loss": 2.0905375480651855, "classification_loss": 0.3060681223869324, "lm_loss": 3.568938732147217, "lr": 5.5200729927007305e-05, "updates_per_second": 6.364, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 123.8, "task": "sst2"}
|
| 35 |
+
{"event": "train", "step": 775, "total_steps": 6315, "epoch_fraction": 0.3682, "loss": 2.221289873123169, "classification_loss": 0.4373878836631775, "lm_loss": 3.5678036212921143, "lr": 5.4952792764201844e-05, "updates_per_second": 6.379, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 127.8, "task": "sst2"}
|
| 36 |
+
{"event": "train", "step": 800, "total_steps": 6315, "epoch_fraction": 0.3801, "loss": 2.1419060230255127, "classification_loss": 0.4003071188926697, "lm_loss": 3.4831981658935547, "lr": 5.470485560139638e-05, "updates_per_second": 6.397, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 131.7, "task": "sst2"}
|
| 37 |
+
{"event": "train", "step": 825, "total_steps": 6315, "epoch_fraction": 0.392, "loss": 2.0366463661193848, "classification_loss": 0.28864139318466187, "lm_loss": 3.4960100650787354, "lr": 5.445691843859092e-05, "updates_per_second": 6.389, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 135.6, "task": "sst2"}
|
| 38 |
+
{"event": "train", "step": 850, "total_steps": 6315, "epoch_fraction": 0.4039, "loss": 2.225525379180908, "classification_loss": 0.4888320863246918, "lm_loss": 3.473386764526367, "lr": 5.420898127578547e-05, "updates_per_second": 6.375, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 139.5, "task": "sst2"}
|
| 39 |
+
{"event": "train", "step": 875, "total_steps": 6315, "epoch_fraction": 0.4157, "loss": 2.0077273845672607, "classification_loss": 0.2892894148826599, "lm_loss": 3.436875581741333, "lr": 5.3961044112980007e-05, "updates_per_second": 6.404, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 143.4, "task": "sst2"}
|
| 40 |
+
{"event": "train", "step": 900, "total_steps": 6315, "epoch_fraction": 0.4276, "loss": 2.0611488819122314, "classification_loss": 0.3252013027667999, "lm_loss": 3.4718949794769287, "lr": 5.3713106950174545e-05, "updates_per_second": 6.327, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 147.4, "task": "sst2"}
|
| 41 |
+
{"event": "train", "step": 925, "total_steps": 6315, "epoch_fraction": 0.4395, "loss": 2.0636043548583984, "classification_loss": 0.34856706857681274, "lm_loss": 3.4300739765167236, "lr": 5.3465169787369084e-05, "updates_per_second": 6.43, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 151.3, "task": "sst2"}
|
| 42 |
+
{"event": "train", "step": 950, "total_steps": 6315, "epoch_fraction": 0.4514, "loss": 2.0712730884552, "classification_loss": 0.3680625557899475, "lm_loss": 3.4064207077026367, "lr": 5.321723262456364e-05, "updates_per_second": 6.393, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 155.2, "task": "sst2"}
|
| 43 |
+
{"event": "train", "step": 975, "total_steps": 6315, "epoch_fraction": 0.4633, "loss": 2.116100311279297, "classification_loss": 0.38334211707115173, "lm_loss": 3.4655160903930664, "lr": 5.2969295461758176e-05, "updates_per_second": 6.337, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 159.1, "task": "sst2"}
|
| 44 |
+
{"event": "train", "step": 1000, "total_steps": 6315, "epoch_fraction": 0.4751, "loss": 1.9500205516815186, "classification_loss": 0.2744311988353729, "lm_loss": 3.3511788845062256, "lr": 5.2721358298952715e-05, "updates_per_second": 6.347, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 163.1, "task": "sst2"}
|
| 45 |
+
{"event": "eval", "step": 1000, "epoch": 0, "offset": 32000, "train": {"correct": 114, "total": 128, "accuracy": 0.890625, "loss": 0.2264169603586197, "gold_label_counts": [65, 63], "majority_accuracy": 0.5078125}, "validation": {"correct": 782, "total": 872, "accuracy": 0.8967889908256881, "loss": 0.2833351194858551, "gold_label_counts": [428, 444], "majority_accuracy": 0.5091743119266054}, "best_step": 1000, "omega_std": 0.6022779941558838, "peak_allocated_gib": 3.836, "sample": {"gold": 1, "predicted": 1}, "elapsed_seconds": 164.1, "task": "sst2"}
|
| 46 |
+
{"event": "train", "step": 1025, "total_steps": 6315, "epoch_fraction": 0.487, "loss": 1.983746886253357, "classification_loss": 0.27982696890830994, "lm_loss": 3.407839775085449, "lr": 5.2473421136147253e-05, "updates_per_second": 6.226, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 170.8, "task": "sst2"}
|
| 47 |
+
{"event": "train", "step": 1050, "total_steps": 6315, "epoch_fraction": 0.4989, "loss": 1.9143515825271606, "classification_loss": 0.27044421434402466, "lm_loss": 3.287814140319824, "lr": 5.22254839733418e-05, "updates_per_second": 6.37, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 174.7, "task": "sst2"}
|
| 48 |
+
{"event": "train", "step": 1075, "total_steps": 6315, "epoch_fraction": 0.5108, "loss": 2.0181663036346436, "classification_loss": 0.3344164490699768, "lm_loss": 3.367499828338623, "lr": 5.197754681053634e-05, "updates_per_second": 6.354, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 178.6, "task": "sst2"}
|
| 49 |
+
{"event": "train", "step": 1100, "total_steps": 6315, "epoch_fraction": 0.5227, "loss": 2.104790687561035, "classification_loss": 0.4090947210788727, "lm_loss": 3.391392230987549, "lr": 5.172960964773088e-05, "updates_per_second": 6.262, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 182.6, "task": "sst2"}
|
| 50 |
+
{"event": "train", "step": 1125, "total_steps": 6315, "epoch_fraction": 0.5345, "loss": 1.952519178390503, "classification_loss": 0.29487964510917664, "lm_loss": 3.3152785301208496, "lr": 5.148167248492542e-05, "updates_per_second": 6.113, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 186.7, "task": "sst2"}
|
| 51 |
+
{"event": "train", "step": 1150, "total_steps": 6315, "epoch_fraction": 0.5464, "loss": 1.9833621978759766, "classification_loss": 0.31395387649536133, "lm_loss": 3.3388173580169678, "lr": 5.123373532211997e-05, "updates_per_second": 6.332, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 190.7, "task": "sst2"}
|
| 52 |
+
{"event": "train", "step": 1175, "total_steps": 6315, "epoch_fraction": 0.5583, "loss": 1.866973638534546, "classification_loss": 0.24248676002025604, "lm_loss": 3.2489736080169678, "lr": 5.098579815931451e-05, "updates_per_second": 6.359, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 194.6, "task": "sst2"}
|
| 53 |
+
{"event": "train", "step": 1200, "total_steps": 6315, "epoch_fraction": 0.5702, "loss": 1.9335218667984009, "classification_loss": 0.2946038246154785, "lm_loss": 3.2778358459472656, "lr": 5.0737860996509046e-05, "updates_per_second": 6.364, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 198.5, "task": "sst2"}
|
| 54 |
+
{"event": "train", "step": 1225, "total_steps": 6315, "epoch_fraction": 0.582, "loss": 1.895715594291687, "classification_loss": 0.2587879002094269, "lm_loss": 3.273855209350586, "lr": 5.0489923833703585e-05, "updates_per_second": 6.354, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 202.5, "task": "sst2"}
|
| 55 |
+
{"event": "train", "step": 1250, "total_steps": 6315, "epoch_fraction": 0.5939, "loss": 1.901862621307373, "classification_loss": 0.26384058594703674, "lm_loss": 3.2760438919067383, "lr": 5.0241986670898124e-05, "updates_per_second": 6.401, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 206.4, "task": "sst2"}
|
| 56 |
+
{"event": "train", "step": 1275, "total_steps": 6315, "epoch_fraction": 0.6058, "loss": 1.8256086111068726, "classification_loss": 0.22038613259792328, "lm_loss": 3.2104451656341553, "lr": 4.999404950809267e-05, "updates_per_second": 6.416, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 210.3, "task": "sst2"}
|
| 57 |
+
{"event": "train", "step": 1300, "total_steps": 6315, "epoch_fraction": 0.6177, "loss": 1.8099651336669922, "classification_loss": 0.22636042535305023, "lm_loss": 3.1672093868255615, "lr": 4.974611234528721e-05, "updates_per_second": 6.405, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 214.2, "task": "sst2"}
|
| 58 |
+
{"event": "train", "step": 1325, "total_steps": 6315, "epoch_fraction": 0.6296, "loss": 1.8013825416564941, "classification_loss": 0.22783803939819336, "lm_loss": 3.1470887660980225, "lr": 4.9498175182481754e-05, "updates_per_second": 6.352, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 218.1, "task": "sst2"}
|
| 59 |
+
{"event": "train", "step": 1350, "total_steps": 6315, "epoch_fraction": 0.6414, "loss": 1.8233660459518433, "classification_loss": 0.27154994010925293, "lm_loss": 3.1036322116851807, "lr": 4.925023801967629e-05, "updates_per_second": 6.403, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 222.0, "task": "sst2"}
|
| 60 |
+
{"event": "train", "step": 1375, "total_steps": 6315, "epoch_fraction": 0.6533, "loss": 1.791058897972107, "classification_loss": 0.24135923385620117, "lm_loss": 3.099400043487549, "lr": 4.900230085687084e-05, "updates_per_second": 6.35, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 225.9, "task": "sst2"}
|
| 61 |
+
{"event": "train", "step": 1400, "total_steps": 6315, "epoch_fraction": 0.6652, "loss": 1.8351759910583496, "classification_loss": 0.2835638225078583, "lm_loss": 3.1032238006591797, "lr": 4.875436369406538e-05, "updates_per_second": 6.398, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 229.9, "task": "sst2"}
|
| 62 |
+
{"event": "train", "step": 1425, "total_steps": 6315, "epoch_fraction": 0.6771, "loss": 1.8302552700042725, "classification_loss": 0.2559477984905243, "lm_loss": 3.1486153602600098, "lr": 4.8506426531259916e-05, "updates_per_second": 6.422, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 233.7, "task": "sst2"}
|
| 63 |
+
{"event": "train", "step": 1450, "total_steps": 6315, "epoch_fraction": 0.6889, "loss": 1.8295344114303589, "classification_loss": 0.2728537917137146, "lm_loss": 3.11336088180542, "lr": 4.8258489368454455e-05, "updates_per_second": 6.366, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 237.7, "task": "sst2"}
|
| 64 |
+
{"event": "train", "step": 1475, "total_steps": 6315, "epoch_fraction": 0.7008, "loss": 1.8054550886154175, "classification_loss": 0.25914525985717773, "lm_loss": 3.0926198959350586, "lr": 4.8010552205649e-05, "updates_per_second": 6.414, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 241.6, "task": "sst2"}
|
| 65 |
+
{"event": "train", "step": 1500, "total_steps": 6315, "epoch_fraction": 0.7127, "loss": 1.7590000629425049, "classification_loss": 0.23628398776054382, "lm_loss": 3.0454323291778564, "lr": 4.7762615042843547e-05, "updates_per_second": 6.493, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 245.4, "task": "sst2"}
|
| 66 |
+
{"event": "train", "step": 1525, "total_steps": 6315, "epoch_fraction": 0.7246, "loss": 1.7761268615722656, "classification_loss": 0.2729516327381134, "lm_loss": 3.0063509941101074, "lr": 4.7514677880038085e-05, "updates_per_second": 4.852, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 250.6, "task": "sst2"}
|
| 67 |
+
{"event": "train", "step": 1550, "total_steps": 6315, "epoch_fraction": 0.7365, "loss": 1.7976841926574707, "classification_loss": 0.23568426072597504, "lm_loss": 3.124000072479248, "lr": 4.7266740717232624e-05, "updates_per_second": 6.522, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 254.4, "task": "sst2"}
|
| 68 |
+
{"event": "train", "step": 1575, "total_steps": 6315, "epoch_fraction": 0.7483, "loss": 1.8113112449645996, "classification_loss": 0.28965499997138977, "lm_loss": 3.0433120727539062, "lr": 4.701880355442717e-05, "updates_per_second": 6.506, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 258.3, "task": "sst2"}
|
| 69 |
+
{"event": "train", "step": 1600, "total_steps": 6315, "epoch_fraction": 0.7602, "loss": 1.7587957382202148, "classification_loss": 0.26970916986465454, "lm_loss": 2.9781734943389893, "lr": 4.677086639162171e-05, "updates_per_second": 6.516, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 262.1, "task": "sst2"}
|
| 70 |
+
{"event": "train", "step": 1625, "total_steps": 6315, "epoch_fraction": 0.7721, "loss": 1.7486045360565186, "classification_loss": 0.24468010663986206, "lm_loss": 3.0078494548797607, "lr": 4.652292922881625e-05, "updates_per_second": 6.452, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 266.0, "task": "sst2"}
|
| 71 |
+
{"event": "train", "step": 1650, "total_steps": 6315, "epoch_fraction": 0.784, "loss": 1.7151767015457153, "classification_loss": 0.22736896574497223, "lm_loss": 2.9756155014038086, "lr": 4.627499206601079e-05, "updates_per_second": 6.502, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 269.8, "task": "sst2"}
|
| 72 |
+
{"event": "train", "step": 1675, "total_steps": 6315, "epoch_fraction": 0.7959, "loss": 1.65114426612854, "classification_loss": 0.20179463922977448, "lm_loss": 2.8986992835998535, "lr": 4.602705490320533e-05, "updates_per_second": 6.482, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 273.7, "task": "sst2"}
|
| 73 |
+
{"event": "train", "step": 1700, "total_steps": 6315, "epoch_fraction": 0.8077, "loss": 1.7247687578201294, "classification_loss": 0.23193511366844177, "lm_loss": 2.9856677055358887, "lr": 4.577911774039988e-05, "updates_per_second": 6.518, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 277.5, "task": "sst2"}
|
| 74 |
+
{"event": "train", "step": 1725, "total_steps": 6315, "epoch_fraction": 0.8196, "loss": 1.689850926399231, "classification_loss": 0.23184241354465485, "lm_loss": 2.9160168170928955, "lr": 4.553118057759442e-05, "updates_per_second": 6.525, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 281.3, "task": "sst2"}
|
| 75 |
+
{"event": "train", "step": 1750, "total_steps": 6315, "epoch_fraction": 0.8315, "loss": 1.6788451671600342, "classification_loss": 0.21702100336551666, "lm_loss": 2.923649549484253, "lr": 4.5283243414788956e-05, "updates_per_second": 6.513, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 285.2, "task": "sst2"}
|
| 76 |
+
{"event": "train", "step": 1775, "total_steps": 6315, "epoch_fraction": 0.8434, "loss": 1.684306263923645, "classification_loss": 0.24582627415657043, "lm_loss": 2.876959800720215, "lr": 4.50353062519835e-05, "updates_per_second": 6.506, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 289.0, "task": "sst2"}
|
| 77 |
+
{"event": "train", "step": 1800, "total_steps": 6315, "epoch_fraction": 0.8552, "loss": 1.6436855792999268, "classification_loss": 0.23176482319831848, "lm_loss": 2.8238415718078613, "lr": 4.478736908917804e-05, "updates_per_second": 6.532, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 292.9, "task": "sst2"}
|
| 78 |
+
{"event": "train", "step": 1825, "total_steps": 6315, "epoch_fraction": 0.8671, "loss": 1.731827974319458, "classification_loss": 0.2530769407749176, "lm_loss": 2.9575021266937256, "lr": 4.453943192637258e-05, "updates_per_second": 6.53, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 296.7, "task": "sst2"}
|
| 79 |
+
{"event": "train", "step": 1850, "total_steps": 6315, "epoch_fraction": 0.879, "loss": 1.6083616018295288, "classification_loss": 0.1730087697505951, "lm_loss": 2.870706081390381, "lr": 4.429149476356712e-05, "updates_per_second": 6.55, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 300.5, "task": "sst2"}
|
| 80 |
+
{"event": "train", "step": 1875, "total_steps": 6315, "epoch_fraction": 0.8909, "loss": 1.712855339050293, "classification_loss": 0.27130603790283203, "lm_loss": 2.8830995559692383, "lr": 4.4043557600761664e-05, "updates_per_second": 6.558, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 304.3, "task": "sst2"}
|
| 81 |
+
{"event": "train", "step": 1900, "total_steps": 6315, "epoch_fraction": 0.9028, "loss": 1.7093693017959595, "classification_loss": 0.2546707093715668, "lm_loss": 2.9093968868255615, "lr": 4.379562043795621e-05, "updates_per_second": 6.557, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 308.1, "task": "sst2"}
|
| 82 |
+
{"event": "train", "step": 1925, "total_steps": 6315, "epoch_fraction": 0.9146, "loss": 1.6174147129058838, "classification_loss": 0.19342784583568573, "lm_loss": 2.8479738235473633, "lr": 4.354768327515075e-05, "updates_per_second": 6.553, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 311.9, "task": "sst2"}
|
| 83 |
+
{"event": "train", "step": 1950, "total_steps": 6315, "epoch_fraction": 0.9265, "loss": 1.6422756910324097, "classification_loss": 0.23898069560527802, "lm_loss": 2.8065903186798096, "lr": 4.329974611234529e-05, "updates_per_second": 6.518, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 315.8, "task": "sst2"}
|
| 84 |
+
{"event": "train", "step": 1975, "total_steps": 6315, "epoch_fraction": 0.9384, "loss": 1.7103263139724731, "classification_loss": 0.29444074630737305, "lm_loss": 2.8317718505859375, "lr": 4.3051808949539826e-05, "updates_per_second": 6.413, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 319.7, "task": "sst2"}
|
| 85 |
+
{"event": "train", "step": 2000, "total_steps": 6315, "epoch_fraction": 0.9503, "loss": 1.5971734523773193, "classification_loss": 0.20693810284137726, "lm_loss": 2.780470848083496, "lr": 4.280387178673437e-05, "updates_per_second": 6.456, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 323.6, "task": "sst2"}
|
| 86 |
+
{"event": "eval", "step": 2000, "epoch": 0, "offset": 64000, "train": {"correct": 126, "total": 128, "accuracy": 0.984375, "loss": 0.051577530801296234, "gold_label_counts": [65, 63], "majority_accuracy": 0.5078125}, "validation": {"correct": 779, "total": 872, "accuracy": 0.893348623853211, "loss": 0.2718764841556549, "gold_label_counts": [428, 444], "majority_accuracy": 0.5091743119266054}, "best_step": 1000, "omega_std": 0.6022703051567078, "peak_allocated_gib": 3.836, "sample": {"gold": 1, "predicted": 1}, "elapsed_seconds": 324.5, "task": "sst2"}
|
| 87 |
+
{"event": "train", "step": 2025, "total_steps": 6315, "epoch_fraction": 0.9622, "loss": 1.6413824558258057, "classification_loss": 0.24593831598758698, "lm_loss": 2.790889024734497, "lr": 4.255593462392891e-05, "updates_per_second": 6.195, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 329.8, "task": "sst2"}
|
| 88 |
+
{"event": "train", "step": 2050, "total_steps": 6315, "epoch_fraction": 0.974, "loss": 1.641890048980713, "classification_loss": 0.25686877965927124, "lm_loss": 2.7700424194335938, "lr": 4.230799746112345e-05, "updates_per_second": 6.39, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 333.7, "task": "sst2"}
|
| 89 |
+
{"event": "train", "step": 2075, "total_steps": 6315, "epoch_fraction": 0.9859, "loss": 1.5833920240402222, "classification_loss": 0.20404170453548431, "lm_loss": 2.758699893951416, "lr": 4.2060060298317995e-05, "updates_per_second": 6.258, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 337.7, "task": "sst2"}
|
| 90 |
+
{"event": "train", "step": 2100, "total_steps": 6315, "epoch_fraction": 0.9978, "loss": 1.4882575273513794, "classification_loss": 0.16907045245170593, "lm_loss": 2.6383743286132812, "lr": 4.181212313551254e-05, "updates_per_second": 6.17, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 341.7, "task": "sst2"}
|
| 91 |
+
{"event": "eval", "step": 2105, "epoch": 1, "offset": 0, "train": {"correct": 125, "total": 128, "accuracy": 0.9765625, "loss": 0.113694928586483, "gold_label_counts": [65, 63], "majority_accuracy": 0.5078125}, "validation": {"correct": 778, "total": 872, "accuracy": 0.8922018348623854, "loss": 0.28741559386253357, "gold_label_counts": [428, 444], "majority_accuracy": 0.5091743119266054}, "best_step": 1000, "omega_std": 0.6022741794586182, "peak_allocated_gib": 3.836, "sample": {"gold": 1, "predicted": 1}, "elapsed_seconds": 343.4, "task": "sst2"}
|
| 92 |
+
{"event": "train", "step": 2125, "total_steps": 6315, "epoch_fraction": 1.0095, "loss": 1.3327664136886597, "classification_loss": 0.1422678530216217, "lm_loss": 2.3809967041015625, "lr": 4.156418597270708e-05, "updates_per_second": 4.143, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 347.8, "task": "sst2"}
|
| 93 |
+
{"event": "train", "step": 2150, "total_steps": 6315, "epoch_fraction": 1.0214, "loss": 1.3208516836166382, "classification_loss": 0.11928025633096695, "lm_loss": 2.4031424522399902, "lr": 4.131624880990162e-05, "updates_per_second": 6.401, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 351.7, "task": "sst2"}
|
| 94 |
+
{"event": "train", "step": 2175, "total_steps": 6315, "epoch_fraction": 1.0333, "loss": 1.2867289781570435, "classification_loss": 0.11750099062919617, "lm_loss": 2.3384554386138916, "lr": 4.106831164709616e-05, "updates_per_second": 6.354, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 355.6, "task": "sst2"}
|
| 95 |
+
{"event": "train", "step": 2200, "total_steps": 6315, "epoch_fraction": 1.0451, "loss": 1.3312835693359375, "classification_loss": 0.15466409921646118, "lm_loss": 2.353238821029663, "lr": 4.08203744842907e-05, "updates_per_second": 6.425, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 359.5, "task": "sst2"}
|
| 96 |
+
{"event": "train", "step": 2225, "total_steps": 6315, "epoch_fraction": 1.057, "loss": 1.3348935842514038, "classification_loss": 0.14646656811237335, "lm_loss": 2.376854419708252, "lr": 4.057243732148524e-05, "updates_per_second": 6.493, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 363.3, "task": "sst2"}
|
| 97 |
+
{"event": "train", "step": 2250, "total_steps": 6315, "epoch_fraction": 1.0689, "loss": 1.3024793863296509, "classification_loss": 0.1188274621963501, "lm_loss": 2.3673038482666016, "lr": 4.032450015867979e-05, "updates_per_second": 6.344, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 367.3, "task": "sst2"}
|
| 98 |
+
{"event": "train", "step": 2275, "total_steps": 6315, "epoch_fraction": 1.0808, "loss": 1.2602726221084595, "classification_loss": 0.09448239952325821, "lm_loss": 2.33158016204834, "lr": 4.007656299587433e-05, "updates_per_second": 6.482, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 371.1, "task": "sst2"}
|
| 99 |
+
{"event": "train", "step": 2300, "total_steps": 6315, "epoch_fraction": 1.0927, "loss": 1.305402159690857, "classification_loss": 0.13467752933502197, "lm_loss": 2.34144926071167, "lr": 3.982862583306887e-05, "updates_per_second": 6.514, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 375.0, "task": "sst2"}
|
| 100 |
+
{"event": "train", "step": 2325, "total_steps": 6315, "epoch_fraction": 1.1045, "loss": 1.3182575702667236, "classification_loss": 0.14482665061950684, "lm_loss": 2.3468618392944336, "lr": 3.958068867026341e-05, "updates_per_second": 6.441, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 378.9, "task": "sst2"}
|
| 101 |
+
{"event": "train", "step": 2350, "total_steps": 6315, "epoch_fraction": 1.1164, "loss": 1.290610909461975, "classification_loss": 0.12146617472171783, "lm_loss": 2.338289737701416, "lr": 3.933275150745795e-05, "updates_per_second": 6.381, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 382.8, "task": "sst2"}
|
| 102 |
+
{"event": "train", "step": 2375, "total_steps": 6315, "epoch_fraction": 1.1283, "loss": 1.358985424041748, "classification_loss": 0.16095858812332153, "lm_loss": 2.3960537910461426, "lr": 3.908481434465249e-05, "updates_per_second": 6.483, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 386.6, "task": "sst2"}
|
| 103 |
+
{"event": "train", "step": 2400, "total_steps": 6315, "epoch_fraction": 1.1402, "loss": 1.237817406654358, "classification_loss": 0.08296418935060501, "lm_loss": 2.309706449508667, "lr": 3.883687718184703e-05, "updates_per_second": 6.471, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 390.5, "task": "sst2"}
|
| 104 |
+
{"event": "train", "step": 2425, "total_steps": 6315, "epoch_fraction": 1.152, "loss": 1.2899526357650757, "classification_loss": 0.12435290217399597, "lm_loss": 2.3311996459960938, "lr": 3.8588940019041574e-05, "updates_per_second": 6.499, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 394.3, "task": "sst2"}
|
| 105 |
+
{"event": "train", "step": 2450, "total_steps": 6315, "epoch_fraction": 1.1639, "loss": 1.3038277626037598, "classification_loss": 0.13939616084098816, "lm_loss": 2.3288626670837402, "lr": 3.834100285623612e-05, "updates_per_second": 6.501, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 398.2, "task": "sst2"}
|
| 106 |
+
{"event": "train", "step": 2475, "total_steps": 6315, "epoch_fraction": 1.1758, "loss": 1.2529693841934204, "classification_loss": 0.10367093980312347, "lm_loss": 2.2985963821411133, "lr": 3.809306569343066e-05, "updates_per_second": 6.362, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 402.1, "task": "sst2"}
|
| 107 |
+
{"event": "train", "step": 2500, "total_steps": 6315, "epoch_fraction": 1.1877, "loss": 1.3048043251037598, "classification_loss": 0.15420624613761902, "lm_loss": 2.3011958599090576, "lr": 3.78451285306252e-05, "updates_per_second": 6.335, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 406.1, "task": "sst2"}
|
| 108 |
+
{"event": "train", "step": 2525, "total_steps": 6315, "epoch_fraction": 1.1996, "loss": 1.277807593345642, "classification_loss": 0.12403827160596848, "lm_loss": 2.3075387477874756, "lr": 3.759719136781974e-05, "updates_per_second": 4.521, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 411.6, "task": "sst2"}
|
| 109 |
+
{"event": "train", "step": 2550, "total_steps": 6315, "epoch_fraction": 1.2114, "loss": 1.2965768575668335, "classification_loss": 0.12494245916604996, "lm_loss": 2.3432681560516357, "lr": 3.734925420501428e-05, "updates_per_second": 6.045, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 415.7, "task": "sst2"}
|
| 110 |
+
{"event": "train", "step": 2575, "total_steps": 6315, "epoch_fraction": 1.2233, "loss": 1.3137949705123901, "classification_loss": 0.15851113200187683, "lm_loss": 2.310567855834961, "lr": 3.710131704220882e-05, "updates_per_second": 6.236, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 419.7, "task": "sst2"}
|
| 111 |
+
{"event": "train", "step": 2600, "total_steps": 6315, "epoch_fraction": 1.2352, "loss": 1.2573726177215576, "classification_loss": 0.10304878652095795, "lm_loss": 2.308647871017456, "lr": 3.685337987940336e-05, "updates_per_second": 6.204, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 423.8, "task": "sst2"}
|
| 112 |
+
{"event": "train", "step": 2625, "total_steps": 6315, "epoch_fraction": 1.2471, "loss": 1.2753835916519165, "classification_loss": 0.1300026774406433, "lm_loss": 2.2907614707946777, "lr": 3.660544271659791e-05, "updates_per_second": 6.336, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 427.7, "task": "sst2"}
|
| 113 |
+
{"event": "train", "step": 2650, "total_steps": 6315, "epoch_fraction": 1.2589, "loss": 1.2620713710784912, "classification_loss": 0.11944767087697983, "lm_loss": 2.285247325897217, "lr": 3.635750555379245e-05, "updates_per_second": 6.358, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 431.7, "task": "sst2"}
|
| 114 |
+
{"event": "train", "step": 2675, "total_steps": 6315, "epoch_fraction": 1.2708, "loss": 1.2695919275283813, "classification_loss": 0.14282947778701782, "lm_loss": 2.2535250186920166, "lr": 3.610956839098699e-05, "updates_per_second": 6.304, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 435.6, "task": "sst2"}
|
| 115 |
+
{"event": "train", "step": 2700, "total_steps": 6315, "epoch_fraction": 1.2827, "loss": 1.28545343875885, "classification_loss": 0.15267477929592133, "lm_loss": 2.2655577659606934, "lr": 3.586163122818153e-05, "updates_per_second": 6.334, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 439.6, "task": "sst2"}
|
| 116 |
+
{"event": "train", "step": 2725, "total_steps": 6315, "epoch_fraction": 1.2946, "loss": 1.2008966207504272, "classification_loss": 0.10350804775953293, "lm_loss": 2.194777488708496, "lr": 3.5613694065376074e-05, "updates_per_second": 6.343, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 443.5, "task": "sst2"}
|
| 117 |
+
{"event": "train", "step": 2750, "total_steps": 6315, "epoch_fraction": 1.3065, "loss": 1.2916909456253052, "classification_loss": 0.15260839462280273, "lm_loss": 2.278165340423584, "lr": 3.536575690257061e-05, "updates_per_second": 6.347, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 447.5, "task": "sst2"}
|
| 118 |
+
{"event": "train", "step": 2775, "total_steps": 6315, "epoch_fraction": 1.3183, "loss": 1.2881431579589844, "classification_loss": 0.1538577824831009, "lm_loss": 2.268570899963379, "lr": 3.511781973976515e-05, "updates_per_second": 6.375, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 451.4, "task": "sst2"}
|
| 119 |
+
{"event": "train", "step": 2800, "total_steps": 6315, "epoch_fraction": 1.3302, "loss": 1.272625207901001, "classification_loss": 0.16515812277793884, "lm_loss": 2.2149341106414795, "lr": 3.486988257695969e-05, "updates_per_second": 6.41, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 455.3, "task": "sst2"}
|
| 120 |
+
{"event": "train", "step": 2825, "total_steps": 6315, "epoch_fraction": 1.3421, "loss": 1.2678741216659546, "classification_loss": 0.1398930549621582, "lm_loss": 2.255962371826172, "lr": 3.462194541415424e-05, "updates_per_second": 6.316, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 459.2, "task": "sst2"}
|
| 121 |
+
{"event": "train", "step": 2850, "total_steps": 6315, "epoch_fraction": 1.354, "loss": 1.3053290843963623, "classification_loss": 0.19181764125823975, "lm_loss": 2.227022886276245, "lr": 3.437400825134878e-05, "updates_per_second": 6.336, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 463.2, "task": "sst2"}
|
| 122 |
+
{"event": "train", "step": 2875, "total_steps": 6315, "epoch_fraction": 1.3659, "loss": 1.2798902988433838, "classification_loss": 0.13484252989292145, "lm_loss": 2.290095090866089, "lr": 3.412607108854332e-05, "updates_per_second": 6.326, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 467.1, "task": "sst2"}
|
| 123 |
+
{"event": "train", "step": 2900, "total_steps": 6315, "epoch_fraction": 1.3777, "loss": 1.2520192861557007, "classification_loss": 0.1509593278169632, "lm_loss": 2.202120065689087, "lr": 3.387813392573786e-05, "updates_per_second": 6.167, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 471.2, "task": "sst2"}
|
| 124 |
+
{"event": "train", "step": 2925, "total_steps": 6315, "epoch_fraction": 1.3896, "loss": 1.2506310939788818, "classification_loss": 0.14834991097450256, "lm_loss": 2.204562187194824, "lr": 3.3630196762932406e-05, "updates_per_second": 6.307, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 475.2, "task": "sst2"}
|
| 125 |
+
{"event": "train", "step": 2950, "total_steps": 6315, "epoch_fraction": 1.4015, "loss": 1.3027729988098145, "classification_loss": 0.16500528156757355, "lm_loss": 2.2755351066589355, "lr": 3.3382259600126945e-05, "updates_per_second": 6.12, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 479.2, "task": "sst2"}
|
| 126 |
+
{"event": "train", "step": 2975, "total_steps": 6315, "epoch_fraction": 1.4134, "loss": 1.2760673761367798, "classification_loss": 0.14420084655284882, "lm_loss": 2.263732671737671, "lr": 3.3134322437321483e-05, "updates_per_second": 6.28, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 483.2, "task": "sst2"}
|
| 127 |
+
{"event": "train", "step": 3000, "total_steps": 6315, "epoch_fraction": 1.4252, "loss": 1.2327897548675537, "classification_loss": 0.13298064470291138, "lm_loss": 2.199618101119995, "lr": 3.288638527451603e-05, "updates_per_second": 6.161, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 487.3, "task": "sst2"}
|
| 128 |
+
{"event": "eval", "step": 3000, "epoch": 1, "offset": 28640, "train": {"correct": 123, "total": 128, "accuracy": 0.9609375, "loss": 0.10956709086894989, "gold_label_counts": [65, 63], "majority_accuracy": 0.5078125}, "validation": {"correct": 779, "total": 872, "accuracy": 0.893348623853211, "loss": 0.29639068245887756, "gold_label_counts": [428, 444], "majority_accuracy": 0.5091743119266054}, "best_step": 1000, "omega_std": 0.6022860407829285, "peak_allocated_gib": 3.836, "sample": {"gold": 1, "predicted": 1}, "elapsed_seconds": 488.3, "task": "sst2"}
|
| 129 |
+
{"event": "train", "step": 3025, "total_steps": 6315, "epoch_fraction": 1.4371, "loss": 1.2238878011703491, "classification_loss": 0.13290804624557495, "lm_loss": 2.181959867477417, "lr": 3.263844811171057e-05, "updates_per_second": 5.955, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 493.8, "task": "sst2"}
|
| 130 |
+
{"event": "train", "step": 3050, "total_steps": 6315, "epoch_fraction": 1.449, "loss": 1.2047467231750488, "classification_loss": 0.09825049340724945, "lm_loss": 2.2129924297332764, "lr": 3.2390510948905114e-05, "updates_per_second": 6.509, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 497.6, "task": "sst2"}
|
| 131 |
+
{"event": "train", "step": 3075, "total_steps": 6315, "epoch_fraction": 1.4609, "loss": 1.2157918214797974, "classification_loss": 0.11516343802213669, "lm_loss": 2.20125675201416, "lr": 3.214257378609965e-05, "updates_per_second": 6.417, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 501.5, "task": "sst2"}
|
| 132 |
+
{"event": "train", "step": 3100, "total_steps": 6315, "epoch_fraction": 1.4728, "loss": 1.222161054611206, "classification_loss": 0.1328936517238617, "lm_loss": 2.178535223007202, "lr": 3.189463662329419e-05, "updates_per_second": 6.463, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 505.4, "task": "sst2"}
|
| 133 |
+
{"event": "train", "step": 3125, "total_steps": 6315, "epoch_fraction": 1.4846, "loss": 1.2188525199890137, "classification_loss": 0.13410907983779907, "lm_loss": 2.1694869995117188, "lr": 3.164669946048873e-05, "updates_per_second": 6.488, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 509.2, "task": "sst2"}
|
| 134 |
+
{"event": "train", "step": 3150, "total_steps": 6315, "epoch_fraction": 1.4965, "loss": 1.2548226118087769, "classification_loss": 0.1534031182527542, "lm_loss": 2.2028393745422363, "lr": 3.1398762297683276e-05, "updates_per_second": 6.501, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 513.1, "task": "sst2"}
|
| 135 |
+
{"event": "train", "step": 3175, "total_steps": 6315, "epoch_fraction": 1.5084, "loss": 1.1906216144561768, "classification_loss": 0.10909386724233627, "lm_loss": 2.163055419921875, "lr": 3.1150825134877815e-05, "updates_per_second": 6.514, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 516.9, "task": "sst2"}
|
| 136 |
+
{"event": "train", "step": 3200, "total_steps": 6315, "epoch_fraction": 1.5203, "loss": 1.2096662521362305, "classification_loss": 0.11577151715755463, "lm_loss": 2.1877896785736084, "lr": 3.090288797207236e-05, "updates_per_second": 6.512, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 520.8, "task": "sst2"}
|
| 137 |
+
{"event": "train", "step": 3225, "total_steps": 6315, "epoch_fraction": 1.5322, "loss": 1.2156476974487305, "classification_loss": 0.1317785382270813, "lm_loss": 2.1677377223968506, "lr": 3.06549508092669e-05, "updates_per_second": 6.503, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 524.6, "task": "sst2"}
|
| 138 |
+
{"event": "train", "step": 3250, "total_steps": 6315, "epoch_fraction": 1.544, "loss": 1.1833833456039429, "classification_loss": 0.1177893579006195, "lm_loss": 2.131187915802002, "lr": 3.040701364646144e-05, "updates_per_second": 6.486, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 528.5, "task": "sst2"}
|
| 139 |
+
{"event": "train", "step": 3275, "total_steps": 6315, "epoch_fraction": 1.5559, "loss": 1.1908997297286987, "classification_loss": 0.13017456233501434, "lm_loss": 2.121450424194336, "lr": 3.0159076483655984e-05, "updates_per_second": 6.449, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 532.3, "task": "sst2"}
|
| 140 |
+
{"event": "train", "step": 3300, "total_steps": 6315, "epoch_fraction": 1.5678, "loss": 1.2036783695220947, "classification_loss": 0.1338944435119629, "lm_loss": 2.1395680904388428, "lr": 2.9911139320850523e-05, "updates_per_second": 6.454, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 536.2, "task": "sst2"}
|
| 141 |
+
{"event": "train", "step": 3325, "total_steps": 6315, "epoch_fraction": 1.5797, "loss": 1.1567437648773193, "classification_loss": 0.1169634759426117, "lm_loss": 2.0795602798461914, "lr": 2.966320215804507e-05, "updates_per_second": 6.223, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 540.2, "task": "sst2"}
|
| 142 |
+
{"event": "train", "step": 3350, "total_steps": 6315, "epoch_fraction": 1.5915, "loss": 1.1849054098129272, "classification_loss": 0.11122431606054306, "lm_loss": 2.14736270904541, "lr": 2.9415264995239607e-05, "updates_per_second": 6.515, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 544.1, "task": "sst2"}
|
| 143 |
+
{"event": "train", "step": 3375, "total_steps": 6315, "epoch_fraction": 1.6034, "loss": 1.1981391906738281, "classification_loss": 0.14070838689804077, "lm_loss": 2.114861488342285, "lr": 2.916732783243415e-05, "updates_per_second": 6.517, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 547.9, "task": "sst2"}
|
| 144 |
+
{"event": "train", "step": 3400, "total_steps": 6315, "epoch_fraction": 1.6153, "loss": 1.2531408071517944, "classification_loss": 0.18242667615413666, "lm_loss": 2.1414284706115723, "lr": 2.891939066962869e-05, "updates_per_second": 6.436, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 551.8, "task": "sst2"}
|
| 145 |
+
{"event": "train", "step": 3425, "total_steps": 6315, "epoch_fraction": 1.6272, "loss": 1.1819523572921753, "classification_loss": 0.12216305732727051, "lm_loss": 2.1195781230926514, "lr": 2.8671453506823234e-05, "updates_per_second": 6.491, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 555.6, "task": "sst2"}
|
| 146 |
+
{"event": "train", "step": 3450, "total_steps": 6315, "epoch_fraction": 1.6391, "loss": 1.1652369499206543, "classification_loss": 0.13760338723659515, "lm_loss": 2.055267095565796, "lr": 2.8423516344017773e-05, "updates_per_second": 6.155, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 559.7, "task": "sst2"}
|
| 147 |
+
{"event": "train", "step": 3475, "total_steps": 6315, "epoch_fraction": 1.6509, "loss": 1.203298568725586, "classification_loss": 0.13244761526584625, "lm_loss": 2.1417016983032227, "lr": 2.8175579181212315e-05, "updates_per_second": 5.97, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 563.9, "task": "sst2"}
|
| 148 |
+
{"event": "train", "step": 3500, "total_steps": 6315, "epoch_fraction": 1.6628, "loss": 1.1692780256271362, "classification_loss": 0.11871542781591415, "lm_loss": 2.101125717163086, "lr": 2.7927642018406854e-05, "updates_per_second": 6.349, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 567.8, "task": "sst2"}
|
| 149 |
+
{"event": "train", "step": 3525, "total_steps": 6315, "epoch_fraction": 1.6747, "loss": 1.2017385959625244, "classification_loss": 0.1456783562898636, "lm_loss": 2.1121201515197754, "lr": 2.7679704855601397e-05, "updates_per_second": 4.764, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 573.1, "task": "sst2"}
|
| 150 |
+
{"event": "train", "step": 3550, "total_steps": 6315, "epoch_fraction": 1.6866, "loss": 1.142551064491272, "classification_loss": 0.08446997404098511, "lm_loss": 2.1161623001098633, "lr": 2.743176769279594e-05, "updates_per_second": 6.283, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 577.1, "task": "sst2"}
|
| 151 |
+
{"event": "train", "step": 3575, "total_steps": 6315, "epoch_fraction": 1.6985, "loss": 1.1866662502288818, "classification_loss": 0.1454901397228241, "lm_loss": 2.08235239982605, "lr": 2.7183830529990478e-05, "updates_per_second": 6.305, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 581.0, "task": "sst2"}
|
| 152 |
+
{"event": "train", "step": 3600, "total_steps": 6315, "epoch_fraction": 1.7103, "loss": 1.1875144243240356, "classification_loss": 0.1463712751865387, "lm_loss": 2.0822863578796387, "lr": 2.693589336718502e-05, "updates_per_second": 6.359, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 585.0, "task": "sst2"}
|
| 153 |
+
{"event": "train", "step": 3625, "total_steps": 6315, "epoch_fraction": 1.7222, "loss": 1.1320091485977173, "classification_loss": 0.0942283496260643, "lm_loss": 2.075561761856079, "lr": 2.6687956204379562e-05, "updates_per_second": 6.252, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 589.0, "task": "sst2"}
|
| 154 |
+
{"event": "train", "step": 3650, "total_steps": 6315, "epoch_fraction": 1.7341, "loss": 1.1385769844055176, "classification_loss": 0.11192885041236877, "lm_loss": 2.053295850753784, "lr": 2.6440019041574105e-05, "updates_per_second": 6.155, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 593.0, "task": "sst2"}
|
| 155 |
+
{"event": "train", "step": 3675, "total_steps": 6315, "epoch_fraction": 1.746, "loss": 1.1611964702606201, "classification_loss": 0.11778843402862549, "lm_loss": 2.08681583404541, "lr": 2.6192081878768643e-05, "updates_per_second": 6.267, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 597.0, "task": "sst2"}
|
| 156 |
+
{"event": "train", "step": 3700, "total_steps": 6315, "epoch_fraction": 1.7578, "loss": 1.1374341249465942, "classification_loss": 0.11674286425113678, "lm_loss": 2.041382312774658, "lr": 2.5944144715963186e-05, "updates_per_second": 6.487, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 600.9, "task": "sst2"}
|
| 157 |
+
{"event": "train", "step": 3725, "total_steps": 6315, "epoch_fraction": 1.7697, "loss": 1.14423406124115, "classification_loss": 0.11544474214315414, "lm_loss": 2.0575788021087646, "lr": 2.5696207553157728e-05, "updates_per_second": 6.498, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 604.7, "task": "sst2"}
|
| 158 |
+
{"event": "train", "step": 3750, "total_steps": 6315, "epoch_fraction": 1.7816, "loss": 1.14613938331604, "classification_loss": 0.1147085577249527, "lm_loss": 2.062861919403076, "lr": 2.544827039035227e-05, "updates_per_second": 6.489, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 608.6, "task": "sst2"}
|
| 159 |
+
{"event": "train", "step": 3775, "total_steps": 6315, "epoch_fraction": 1.7935, "loss": 1.1526753902435303, "classification_loss": 0.11487869173288345, "lm_loss": 2.0755929946899414, "lr": 2.520033322754681e-05, "updates_per_second": 6.169, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 612.6, "task": "sst2"}
|
| 160 |
+
{"event": "train", "step": 3800, "total_steps": 6315, "epoch_fraction": 1.8054, "loss": 1.1236519813537598, "classification_loss": 0.11324584484100342, "lm_loss": 2.020812511444092, "lr": 2.4952396064741355e-05, "updates_per_second": 6.451, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 616.5, "task": "sst2"}
|
| 161 |
+
{"event": "train", "step": 3825, "total_steps": 6315, "epoch_fraction": 1.8172, "loss": 1.096146583557129, "classification_loss": 0.09824828058481216, "lm_loss": 1.99579656124115, "lr": 2.4704458901935894e-05, "updates_per_second": 6.469, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 620.4, "task": "sst2"}
|
| 162 |
+
{"event": "train", "step": 3850, "total_steps": 6315, "epoch_fraction": 1.8291, "loss": 1.1546887159347534, "classification_loss": 0.12108569592237473, "lm_loss": 2.0672056674957275, "lr": 2.4456521739130436e-05, "updates_per_second": 6.368, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 624.3, "task": "sst2"}
|
| 163 |
+
{"event": "train", "step": 3875, "total_steps": 6315, "epoch_fraction": 1.841, "loss": 1.1352051496505737, "classification_loss": 0.12036287039518356, "lm_loss": 2.0296847820281982, "lr": 2.4208584576324975e-05, "updates_per_second": 6.204, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 628.3, "task": "sst2"}
|
| 164 |
+
{"event": "train", "step": 3900, "total_steps": 6315, "epoch_fraction": 1.8529, "loss": 1.158746361732483, "classification_loss": 0.11048508435487747, "lm_loss": 2.096522569656372, "lr": 2.396064741351952e-05, "updates_per_second": 6.286, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 632.3, "task": "sst2"}
|
| 165 |
+
{"event": "train", "step": 3925, "total_steps": 6315, "epoch_fraction": 1.8647, "loss": 1.114076018333435, "classification_loss": 0.09196887165307999, "lm_loss": 2.0442140102386475, "lr": 2.371271025071406e-05, "updates_per_second": 6.041, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 636.4, "task": "sst2"}
|
| 166 |
+
{"event": "train", "step": 3950, "total_steps": 6315, "epoch_fraction": 1.8766, "loss": 1.1481815576553345, "classification_loss": 0.15175190567970276, "lm_loss": 1.9928597211837769, "lr": 2.3464773087908602e-05, "updates_per_second": 6.276, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 640.4, "task": "sst2"}
|
| 167 |
+
{"event": "train", "step": 3975, "total_steps": 6315, "epoch_fraction": 1.8885, "loss": 1.1204367876052856, "classification_loss": 0.08759386092424393, "lm_loss": 2.065685749053955, "lr": 2.321683592510314e-05, "updates_per_second": 6.252, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 644.4, "task": "sst2"}
|
| 168 |
+
{"event": "train", "step": 4000, "total_steps": 6315, "epoch_fraction": 1.9004, "loss": 1.165933609008789, "classification_loss": 0.17023074626922607, "lm_loss": 1.9914060831069946, "lr": 2.2968898762297686e-05, "updates_per_second": 6.289, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 648.4, "task": "sst2"}
|
| 169 |
+
{"event": "eval", "step": 4000, "epoch": 1, "offset": 60640, "train": {"correct": 125, "total": 128, "accuracy": 0.9765625, "loss": 0.06635785102844238, "gold_label_counts": [65, 63], "majority_accuracy": 0.5078125}, "validation": {"correct": 776, "total": 872, "accuracy": 0.8899082568807339, "loss": 0.27543023228645325, "gold_label_counts": [428, 444], "majority_accuracy": 0.5091743119266054}, "best_step": 1000, "omega_std": 0.6022989153862, "peak_allocated_gib": 3.836, "sample": {"gold": 1, "predicted": 1}, "elapsed_seconds": 649.4, "task": "sst2"}
|
| 170 |
+
{"event": "train", "step": 4025, "total_steps": 6315, "epoch_fraction": 1.9123, "loss": 1.131172776222229, "classification_loss": 0.11088116466999054, "lm_loss": 2.0405831336975098, "lr": 2.2720961599492225e-05, "updates_per_second": 6.223, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 654.8, "task": "sst2"}
|
| 171 |
+
{"event": "train", "step": 4050, "total_steps": 6315, "epoch_fraction": 1.9241, "loss": 1.1471199989318848, "classification_loss": 0.14487981796264648, "lm_loss": 2.0044806003570557, "lr": 2.2473024436686767e-05, "updates_per_second": 6.099, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 658.9, "task": "sst2"}
|
| 172 |
+
{"event": "train", "step": 4075, "total_steps": 6315, "epoch_fraction": 1.936, "loss": 1.0817447900772095, "classification_loss": 0.0961829274892807, "lm_loss": 1.9711235761642456, "lr": 2.2225087273881306e-05, "updates_per_second": 6.267, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 662.9, "task": "sst2"}
|
| 173 |
+
{"event": "train", "step": 4100, "total_steps": 6315, "epoch_fraction": 1.9479, "loss": 1.1521180868148804, "classification_loss": 0.1387293040752411, "lm_loss": 2.0267772674560547, "lr": 2.197715011107585e-05, "updates_per_second": 6.238, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 666.9, "task": "sst2"}
|
| 174 |
+
{"event": "train", "step": 4125, "total_steps": 6315, "epoch_fraction": 1.9598, "loss": 1.1056588888168335, "classification_loss": 0.10961487889289856, "lm_loss": 1.992087960243225, "lr": 2.172921294827039e-05, "updates_per_second": 6.256, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 670.9, "task": "sst2"}
|
| 175 |
+
{"event": "train", "step": 4150, "total_steps": 6315, "epoch_fraction": 1.9717, "loss": 1.1597718000411987, "classification_loss": 0.14887970685958862, "lm_loss": 2.0217843055725098, "lr": 2.148127578546493e-05, "updates_per_second": 6.276, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 674.9, "task": "sst2"}
|
| 176 |
+
{"event": "train", "step": 4175, "total_steps": 6315, "epoch_fraction": 1.9835, "loss": 1.1372652053833008, "classification_loss": 0.11148121953010559, "lm_loss": 2.051568031311035, "lr": 2.1233338622659475e-05, "updates_per_second": 6.05, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 679.0, "task": "sst2"}
|
| 177 |
+
{"event": "train", "step": 4200, "total_steps": 6315, "epoch_fraction": 1.9954, "loss": 1.1025891304016113, "classification_loss": 0.1210303008556366, "lm_loss": 1.9631174802780151, "lr": 2.0985401459854014e-05, "updates_per_second": 6.24, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 683.0, "task": "sst2"}
|
| 178 |
+
{"event": "eval", "step": 4210, "epoch": 2, "offset": 0, "train": {"correct": 126, "total": 128, "accuracy": 0.984375, "loss": 0.03742062300443649, "gold_label_counts": [65, 63], "majority_accuracy": 0.5078125}, "validation": {"correct": 772, "total": 872, "accuracy": 0.8853211009174312, "loss": 0.29878267645835876, "gold_label_counts": [428, 444], "majority_accuracy": 0.5091743119266054}, "best_step": 1000, "omega_std": 0.6023008227348328, "peak_allocated_gib": 3.836, "sample": {"gold": 1, "predicted": 1}, "elapsed_seconds": 685.6, "task": "sst2"}
|
| 179 |
+
{"event": "train", "step": 4225, "total_steps": 6315, "epoch_fraction": 2.0071, "loss": 1.0202771425247192, "classification_loss": 0.0920475572347641, "lm_loss": 1.8564592599868774, "lr": 2.0737464297048557e-05, "updates_per_second": 3.975, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 689.3, "task": "sst2"}
|
| 180 |
+
{"event": "train", "step": 4250, "total_steps": 6315, "epoch_fraction": 2.019, "loss": 0.8784632086753845, "classification_loss": 0.05271118879318237, "lm_loss": 1.6515036821365356, "lr": 2.0489527134243096e-05, "updates_per_second": 6.394, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 693.2, "task": "sst2"}
|
| 181 |
+
{"event": "train", "step": 4275, "total_steps": 6315, "epoch_fraction": 2.0309, "loss": 0.9181175827980042, "classification_loss": 0.06463732570409775, "lm_loss": 1.7069604396820068, "lr": 2.024158997143764e-05, "updates_per_second": 6.461, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 697.1, "task": "sst2"}
|
| 182 |
+
{"event": "train", "step": 4300, "total_steps": 6315, "epoch_fraction": 2.0428, "loss": 0.9381953477859497, "classification_loss": 0.06567905098199844, "lm_loss": 1.7450324296951294, "lr": 1.999365280863218e-05, "updates_per_second": 6.299, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 701.1, "task": "sst2"}
|
| 183 |
+
{"event": "train", "step": 4325, "total_steps": 6315, "epoch_fraction": 2.0546, "loss": 0.8884463310241699, "classification_loss": 0.05492797866463661, "lm_loss": 1.6670366525650024, "lr": 1.9745715645826722e-05, "updates_per_second": 6.234, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 705.1, "task": "sst2"}
|
| 184 |
+
{"event": "train", "step": 4350, "total_steps": 6315, "epoch_fraction": 2.0665, "loss": 0.9547043442726135, "classification_loss": 0.10079241544008255, "lm_loss": 1.7078242301940918, "lr": 1.949777848302126e-05, "updates_per_second": 6.101, "peak_allocated_gib": 3.836, "nonfinite_updates": 0, "elapsed_seconds": 709.2, "task": "sst2"}
|
| 185 |
+
{"event": "start", "step": 4210, "train_examples": 67349, "epochs": 3, "total_steps": 6315, "microbatch": 16, "effective_batch": 32, "memory_cap_gib": 9.0, "trainable_parameters": 123699458, "device": "NVIDIA GeForce RTX 5060 Ti", "task": "sst2"}
|
| 186 |
+
{"event": "train", "step": 4225, "total_steps": 6315, "epoch_fraction": 2.0071, "loss": 0.9257487654685974, "classification_loss": 0.06507761776447296, "lm_loss": 1.7213423252105713, "lr": 2.0737464297048557e-05, "updates_per_second": 4.191, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 689.2, "task": "sst2"}
|
| 187 |
+
{"event": "train", "step": 4250, "total_steps": 6315, "epoch_fraction": 2.019, "loss": 0.8784632086753845, "classification_loss": 0.05271118879318237, "lm_loss": 1.6515036821365356, "lr": 2.0489527134243096e-05, "updates_per_second": 6.34, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 693.1, "task": "sst2"}
|
| 188 |
+
{"event": "train", "step": 4275, "total_steps": 6315, "epoch_fraction": 2.0309, "loss": 0.9181175827980042, "classification_loss": 0.06463732570409775, "lm_loss": 1.7069604396820068, "lr": 2.024158997143764e-05, "updates_per_second": 6.366, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 697.0, "task": "sst2"}
|
| 189 |
+
{"event": "train", "step": 4300, "total_steps": 6315, "epoch_fraction": 2.0428, "loss": 0.9381953477859497, "classification_loss": 0.06567905098199844, "lm_loss": 1.7450324296951294, "lr": 1.999365280863218e-05, "updates_per_second": 6.401, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 700.9, "task": "sst2"}
|
| 190 |
+
{"event": "train", "step": 4325, "total_steps": 6315, "epoch_fraction": 2.0546, "loss": 0.8884463310241699, "classification_loss": 0.05492797866463661, "lm_loss": 1.6670366525650024, "lr": 1.9745715645826722e-05, "updates_per_second": 6.396, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 704.9, "task": "sst2"}
|
| 191 |
+
{"event": "train", "step": 4350, "total_steps": 6315, "epoch_fraction": 2.0665, "loss": 0.9547043442726135, "classification_loss": 0.10079241544008255, "lm_loss": 1.7078242301940918, "lr": 1.949777848302126e-05, "updates_per_second": 6.393, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 708.8, "task": "sst2"}
|
| 192 |
+
{"event": "train", "step": 4375, "total_steps": 6315, "epoch_fraction": 2.0784, "loss": 0.9021185636520386, "classification_loss": 0.04738934710621834, "lm_loss": 1.709458589553833, "lr": 1.9249841320215807e-05, "updates_per_second": 6.217, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 712.8, "task": "sst2"}
|
| 193 |
+
{"event": "train", "step": 4400, "total_steps": 6315, "epoch_fraction": 2.0903, "loss": 0.8922727108001709, "classification_loss": 0.051551952958106995, "lm_loss": 1.6814411878585815, "lr": 1.9001904157410346e-05, "updates_per_second": 6.28, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 716.8, "task": "sst2"}
|
| 194 |
+
{"event": "train", "step": 4425, "total_steps": 6315, "epoch_fraction": 2.1022, "loss": 0.91530841588974, "classification_loss": 0.06731647998094559, "lm_loss": 1.695984125137329, "lr": 1.8753966994604888e-05, "updates_per_second": 6.376, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 720.7, "task": "sst2"}
|
| 195 |
+
{"event": "train", "step": 4450, "total_steps": 6315, "epoch_fraction": 2.114, "loss": 0.9242910742759705, "classification_loss": 0.05029928684234619, "lm_loss": 1.7479838132858276, "lr": 1.8506029831799427e-05, "updates_per_second": 6.382, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 724.6, "task": "sst2"}
|
| 196 |
+
{"event": "train", "step": 4475, "total_steps": 6315, "epoch_fraction": 2.1259, "loss": 0.9030434489250183, "classification_loss": 0.05504282936453819, "lm_loss": 1.6960010528564453, "lr": 1.8258092668993973e-05, "updates_per_second": 6.419, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 728.5, "task": "sst2"}
|
| 197 |
+
{"event": "train", "step": 4500, "total_steps": 6315, "epoch_fraction": 2.1378, "loss": 0.894266664981842, "classification_loss": 0.052005570381879807, "lm_loss": 1.6845221519470215, "lr": 1.801015550618851e-05, "updates_per_second": 6.375, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 732.4, "task": "sst2"}
|
| 198 |
+
{"event": "train", "step": 4525, "total_steps": 6315, "epoch_fraction": 2.1497, "loss": 0.9081729650497437, "classification_loss": 0.05064784735441208, "lm_loss": 1.7150503396987915, "lr": 1.7762218343383054e-05, "updates_per_second": 4.783, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 737.7, "task": "sst2"}
|
| 199 |
+
{"event": "train", "step": 4550, "total_steps": 6315, "epoch_fraction": 2.1615, "loss": 0.9251307249069214, "classification_loss": 0.05903540179133415, "lm_loss": 1.7321903705596924, "lr": 1.7514281180577596e-05, "updates_per_second": 6.281, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 741.6, "task": "sst2"}
|
| 200 |
+
{"event": "train", "step": 4575, "total_steps": 6315, "epoch_fraction": 2.1734, "loss": 0.8640226721763611, "classification_loss": 0.03129173815250397, "lm_loss": 1.6654621362686157, "lr": 1.726634401777214e-05, "updates_per_second": 6.335, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 745.6, "task": "sst2"}
|
| 201 |
+
{"event": "train", "step": 4600, "total_steps": 6315, "epoch_fraction": 2.1853, "loss": 0.8994913101196289, "classification_loss": 0.0453968271613121, "lm_loss": 1.7081890106201172, "lr": 1.7018406854966677e-05, "updates_per_second": 6.153, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 749.6, "task": "sst2"}
|
| 202 |
+
{"event": "train", "step": 4625, "total_steps": 6315, "epoch_fraction": 2.1972, "loss": 0.9195772409439087, "classification_loss": 0.0514405220746994, "lm_loss": 1.7362737655639648, "lr": 1.677046969216122e-05, "updates_per_second": 6.24, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 753.7, "task": "sst2"}
|
| 203 |
+
{"event": "train", "step": 4650, "total_steps": 6315, "epoch_fraction": 2.2091, "loss": 0.8795856833457947, "classification_loss": 0.02909711003303528, "lm_loss": 1.7009778022766113, "lr": 1.6522532529355762e-05, "updates_per_second": 6.305, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 757.6, "task": "sst2"}
|
| 204 |
+
{"event": "train", "step": 4675, "total_steps": 6315, "epoch_fraction": 2.2209, "loss": 0.8984683156013489, "classification_loss": 0.06235487759113312, "lm_loss": 1.6722270250320435, "lr": 1.62745953665503e-05, "updates_per_second": 6.274, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 761.6, "task": "sst2"}
|
| 205 |
+
{"event": "train", "step": 4700, "total_steps": 6315, "epoch_fraction": 2.2328, "loss": 0.8711918592453003, "classification_loss": 0.035346318036317825, "lm_loss": 1.6716912984848022, "lr": 1.6026658203744843e-05, "updates_per_second": 6.343, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 765.6, "task": "sst2"}
|
| 206 |
+
{"event": "train", "step": 4725, "total_steps": 6315, "epoch_fraction": 2.2447, "loss": 0.916427493095398, "classification_loss": 0.05179231986403465, "lm_loss": 1.7292704582214355, "lr": 1.5778721040939382e-05, "updates_per_second": 6.424, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 769.4, "task": "sst2"}
|
| 207 |
+
{"event": "train", "step": 4750, "total_steps": 6315, "epoch_fraction": 2.2566, "loss": 0.882084310054779, "classification_loss": 0.04434753954410553, "lm_loss": 1.675473928451538, "lr": 1.5530783878133928e-05, "updates_per_second": 6.434, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 773.3, "task": "sst2"}
|
| 208 |
+
{"event": "train", "step": 4775, "total_steps": 6315, "epoch_fraction": 2.2685, "loss": 0.9168816804885864, "classification_loss": 0.05906403437256813, "lm_loss": 1.715634822845459, "lr": 1.5282846715328466e-05, "updates_per_second": 6.37, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 777.3, "task": "sst2"}
|
| 209 |
+
{"event": "train", "step": 4800, "total_steps": 6315, "epoch_fraction": 2.2803, "loss": 0.9020529985427856, "classification_loss": 0.06577792018651962, "lm_loss": 1.6725499629974365, "lr": 1.5034909552523009e-05, "updates_per_second": 6.344, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 781.2, "task": "sst2"}
|
| 210 |
+
{"event": "train", "step": 4825, "total_steps": 6315, "epoch_fraction": 2.2922, "loss": 0.9033777713775635, "classification_loss": 0.055071376264095306, "lm_loss": 1.6966127157211304, "lr": 1.4786972389717551e-05, "updates_per_second": 6.462, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 785.1, "task": "sst2"}
|
| 211 |
+
{"event": "train", "step": 4850, "total_steps": 6315, "epoch_fraction": 2.3041, "loss": 0.9213827848434448, "classification_loss": 0.05222030356526375, "lm_loss": 1.7383249998092651, "lr": 1.4539035226912092e-05, "updates_per_second": 6.403, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 789.0, "task": "sst2"}
|
| 212 |
+
{"event": "train", "step": 4875, "total_steps": 6315, "epoch_fraction": 2.316, "loss": 0.9134747385978699, "classification_loss": 0.0647503063082695, "lm_loss": 1.69744873046875, "lr": 1.4291098064106634e-05, "updates_per_second": 6.204, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 793.0, "task": "sst2"}
|
| 213 |
+
{"event": "train", "step": 4900, "total_steps": 6315, "epoch_fraction": 2.3278, "loss": 0.9106994271278381, "classification_loss": 0.05368528887629509, "lm_loss": 1.714028000831604, "lr": 1.4043160901301176e-05, "updates_per_second": 6.427, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 796.9, "task": "sst2"}
|
| 214 |
+
{"event": "train", "step": 4925, "total_steps": 6315, "epoch_fraction": 2.3397, "loss": 0.9281379580497742, "classification_loss": 0.06612811982631683, "lm_loss": 1.7240197658538818, "lr": 1.3795223738495715e-05, "updates_per_second": 6.412, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 800.8, "task": "sst2"}
|
| 215 |
+
{"event": "train", "step": 4950, "total_steps": 6315, "epoch_fraction": 2.3516, "loss": 0.8881926536560059, "classification_loss": 0.038796499371528625, "lm_loss": 1.6987923383712769, "lr": 1.3547286575690257e-05, "updates_per_second": 6.419, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 804.7, "task": "sst2"}
|
| 216 |
+
{"event": "train", "step": 4975, "total_steps": 6315, "epoch_fraction": 2.3635, "loss": 0.9013857841491699, "classification_loss": 0.03821098059415817, "lm_loss": 1.7263495922088623, "lr": 1.3299349412884798e-05, "updates_per_second": 6.366, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 808.6, "task": "sst2"}
|
| 217 |
+
{"event": "train", "step": 5000, "total_steps": 6315, "epoch_fraction": 2.3754, "loss": 0.9025458097457886, "classification_loss": 0.03570696339011192, "lm_loss": 1.733677625656128, "lr": 1.305141225007934e-05, "updates_per_second": 6.38, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 812.5, "task": "sst2"}
|
| 218 |
+
{"event": "eval", "step": 5000, "epoch": 2, "offset": 25280, "train": {"correct": 126, "total": 128, "accuracy": 0.984375, "loss": 0.03098107874393463, "gold_label_counts": [65, 63], "majority_accuracy": 0.5078125}, "validation": {"correct": 782, "total": 872, "accuracy": 0.8967889908256881, "loss": 0.3260447084903717, "gold_label_counts": [428, 444], "majority_accuracy": 0.5091743119266054}, "best_step": 1000, "omega_std": 0.6023032069206238, "peak_allocated_gib": 3.846, "sample": {"gold": 1, "predicted": 1}, "elapsed_seconds": 813.6, "task": "sst2"}
|
| 219 |
+
{"event": "train", "step": 5025, "total_steps": 6315, "epoch_fraction": 2.3872, "loss": 0.8801103234291077, "classification_loss": 0.039739564061164856, "lm_loss": 1.6807414293289185, "lr": 1.280347508727388e-05, "updates_per_second": 6.196, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 819.1, "task": "sst2"}
|
| 220 |
+
{"event": "train", "step": 5050, "total_steps": 6315, "epoch_fraction": 2.3991, "loss": 0.907342255115509, "classification_loss": 0.0556475967168808, "lm_loss": 1.703389286994934, "lr": 1.2555537924468423e-05, "updates_per_second": 6.166, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 823.2, "task": "sst2"}
|
| 221 |
+
{"event": "train", "step": 5075, "total_steps": 6315, "epoch_fraction": 2.411, "loss": 0.8978522419929504, "classification_loss": 0.0486806221306324, "lm_loss": 1.698343276977539, "lr": 1.2307600761662964e-05, "updates_per_second": 6.027, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 827.3, "task": "sst2"}
|
| 222 |
+
{"event": "train", "step": 5100, "total_steps": 6315, "epoch_fraction": 2.4229, "loss": 0.883952796459198, "classification_loss": 0.03924911841750145, "lm_loss": 1.6894073486328125, "lr": 1.2059663598857506e-05, "updates_per_second": 6.174, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 831.4, "task": "sst2"}
|
| 223 |
+
{"event": "train", "step": 5125, "total_steps": 6315, "epoch_fraction": 2.4348, "loss": 0.8547297716140747, "classification_loss": 0.029618248343467712, "lm_loss": 1.6502230167388916, "lr": 1.1811726436052046e-05, "updates_per_second": 6.267, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 835.4, "task": "sst2"}
|
| 224 |
+
{"event": "train", "step": 5150, "total_steps": 6315, "epoch_fraction": 2.4466, "loss": 0.9055683016777039, "classification_loss": 0.05554046854376793, "lm_loss": 1.7000558376312256, "lr": 1.1563789273246589e-05, "updates_per_second": 6.14, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 839.4, "task": "sst2"}
|
| 225 |
+
{"event": "train", "step": 5175, "total_steps": 6315, "epoch_fraction": 2.4585, "loss": 0.9287992715835571, "classification_loss": 0.0689379945397377, "lm_loss": 1.7197223901748657, "lr": 1.131585211044113e-05, "updates_per_second": 6.086, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 843.6, "task": "sst2"}
|
| 226 |
+
{"event": "train", "step": 5200, "total_steps": 6315, "epoch_fraction": 2.4704, "loss": 0.920332670211792, "classification_loss": 0.07521222531795502, "lm_loss": 1.690240740776062, "lr": 1.1067914947635672e-05, "updates_per_second": 6.143, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 847.6, "task": "sst2"}
|
| 227 |
+
{"event": "train", "step": 5225, "total_steps": 6315, "epoch_fraction": 2.4823, "loss": 0.8864288330078125, "classification_loss": 0.03537824749946594, "lm_loss": 1.7021011114120483, "lr": 1.0819977784830212e-05, "updates_per_second": 6.223, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 851.6, "task": "sst2"}
|
| 228 |
+
{"event": "train", "step": 5250, "total_steps": 6315, "epoch_fraction": 2.4941, "loss": 0.9040237069129944, "classification_loss": 0.07675249874591827, "lm_loss": 1.6545428037643433, "lr": 1.0572040622024754e-05, "updates_per_second": 6.275, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 855.6, "task": "sst2"}
|
| 229 |
+
{"event": "train", "step": 5275, "total_steps": 6315, "epoch_fraction": 2.506, "loss": 0.8797132968902588, "classification_loss": 0.045493364334106445, "lm_loss": 1.6684397459030151, "lr": 1.0324103459219295e-05, "updates_per_second": 6.248, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 859.6, "task": "sst2"}
|
| 230 |
+
{"event": "train", "step": 5300, "total_steps": 6315, "epoch_fraction": 2.5179, "loss": 0.903615415096283, "classification_loss": 0.06512227654457092, "lm_loss": 1.6769862174987793, "lr": 1.0076166296413837e-05, "updates_per_second": 6.077, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 863.7, "task": "sst2"}
|
| 231 |
+
{"event": "train", "step": 5325, "total_steps": 6315, "epoch_fraction": 2.5298, "loss": 0.8759164214134216, "classification_loss": 0.05672887712717056, "lm_loss": 1.6383750438690186, "lr": 9.82822913360838e-06, "updates_per_second": 6.285, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 867.7, "task": "sst2"}
|
| 232 |
+
{"event": "train", "step": 5350, "total_steps": 6315, "epoch_fraction": 2.5417, "loss": 0.8723488450050354, "classification_loss": 0.03312929347157478, "lm_loss": 1.6784390211105347, "lr": 9.58029197080292e-06, "updates_per_second": 6.297, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 871.7, "task": "sst2"}
|
| 233 |
+
{"event": "train", "step": 5375, "total_steps": 6315, "epoch_fraction": 2.5535, "loss": 0.8796205520629883, "classification_loss": 0.03368881344795227, "lm_loss": 1.6918635368347168, "lr": 9.332354807997462e-06, "updates_per_second": 6.252, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 875.7, "task": "sst2"}
|
| 234 |
+
{"event": "train", "step": 5400, "total_steps": 6315, "epoch_fraction": 2.5654, "loss": 0.8988900780677795, "classification_loss": 0.055363740772008896, "lm_loss": 1.6870523691177368, "lr": 9.084417645192003e-06, "updates_per_second": 6.367, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 879.6, "task": "sst2"}
|
| 235 |
+
{"event": "train", "step": 5425, "total_steps": 6315, "epoch_fraction": 2.5773, "loss": 0.8943321704864502, "classification_loss": 0.04795122519135475, "lm_loss": 1.692761778831482, "lr": 8.836480482386545e-06, "updates_per_second": 6.342, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 883.6, "task": "sst2"}
|
| 236 |
+
{"event": "train", "step": 5450, "total_steps": 6315, "epoch_fraction": 2.5892, "loss": 0.8663709163665771, "classification_loss": 0.041141804307699203, "lm_loss": 1.6504583358764648, "lr": 8.588543319581086e-06, "updates_per_second": 6.389, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 887.5, "task": "sst2"}
|
| 237 |
+
{"event": "train", "step": 5475, "total_steps": 6315, "epoch_fraction": 2.601, "loss": 0.8623360395431519, "classification_loss": 0.029624156653881073, "lm_loss": 1.66542387008667, "lr": 8.340606156775628e-06, "updates_per_second": 6.378, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 891.4, "task": "sst2"}
|
| 238 |
+
{"event": "train", "step": 5500, "total_steps": 6315, "epoch_fraction": 2.6129, "loss": 0.8800361156463623, "classification_loss": 0.05322343111038208, "lm_loss": 1.6536256074905396, "lr": 8.092668993970167e-06, "updates_per_second": 6.264, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 895.4, "task": "sst2"}
|
| 239 |
+
{"event": "train", "step": 5525, "total_steps": 6315, "epoch_fraction": 2.6248, "loss": 0.9106634855270386, "classification_loss": 0.08137666434049606, "lm_loss": 1.6585736274719238, "lr": 7.84473183116471e-06, "updates_per_second": 4.61, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 900.8, "task": "sst2"}
|
| 240 |
+
{"event": "train", "step": 5550, "total_steps": 6315, "epoch_fraction": 2.6367, "loss": 0.8735886812210083, "classification_loss": 0.05031789839267731, "lm_loss": 1.6465420722961426, "lr": 7.596794668359252e-06, "updates_per_second": 6.376, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 904.7, "task": "sst2"}
|
| 241 |
+
{"event": "train", "step": 5575, "total_steps": 6315, "epoch_fraction": 2.6486, "loss": 0.8849865794181824, "classification_loss": 0.05382001772522926, "lm_loss": 1.6623331308364868, "lr": 7.348857505553793e-06, "updates_per_second": 6.37, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 908.7, "task": "sst2"}
|
| 242 |
+
{"event": "train", "step": 5600, "total_steps": 6315, "epoch_fraction": 2.6604, "loss": 0.8404571413993835, "classification_loss": 0.05492052435874939, "lm_loss": 1.571073293685913, "lr": 7.1009203427483345e-06, "updates_per_second": 6.381, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 912.6, "task": "sst2"}
|
| 243 |
+
{"event": "train", "step": 5625, "total_steps": 6315, "epoch_fraction": 2.6723, "loss": 0.8786718845367432, "classification_loss": 0.06807456165552139, "lm_loss": 1.62119460105896, "lr": 6.852983179942876e-06, "updates_per_second": 6.397, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 916.5, "task": "sst2"}
|
| 244 |
+
{"event": "train", "step": 5650, "total_steps": 6315, "epoch_fraction": 2.6842, "loss": 0.860595166683197, "classification_loss": 0.02986852638423443, "lm_loss": 1.6614534854888916, "lr": 6.6050460171374165e-06, "updates_per_second": 6.421, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 920.4, "task": "sst2"}
|
| 245 |
+
{"event": "train", "step": 5675, "total_steps": 6315, "epoch_fraction": 2.6961, "loss": 0.8735769391059875, "classification_loss": 0.04739278554916382, "lm_loss": 1.6523680686950684, "lr": 6.357108854331958e-06, "updates_per_second": 6.366, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 924.3, "task": "sst2"}
|
| 246 |
+
{"event": "train", "step": 5700, "total_steps": 6315, "epoch_fraction": 2.708, "loss": 0.8620420098304749, "classification_loss": 0.03537702560424805, "lm_loss": 1.6533300876617432, "lr": 6.109171691526499e-06, "updates_per_second": 6.192, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 928.4, "task": "sst2"}
|
| 247 |
+
{"event": "train", "step": 5725, "total_steps": 6315, "epoch_fraction": 2.7198, "loss": 0.8813208341598511, "classification_loss": 0.056214842945337296, "lm_loss": 1.650212049484253, "lr": 5.861234528721041e-06, "updates_per_second": 6.122, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 932.4, "task": "sst2"}
|
| 248 |
+
{"event": "train", "step": 5750, "total_steps": 6315, "epoch_fraction": 2.7317, "loss": 0.8369200229644775, "classification_loss": 0.04046132043004036, "lm_loss": 1.5929170846939087, "lr": 5.613297365915582e-06, "updates_per_second": 6.094, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 936.5, "task": "sst2"}
|
| 249 |
+
{"event": "train", "step": 5775, "total_steps": 6315, "epoch_fraction": 2.7436, "loss": 0.8754738569259644, "classification_loss": 0.03492119535803795, "lm_loss": 1.6811054944992065, "lr": 5.365360203110124e-06, "updates_per_second": 6.295, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 940.5, "task": "sst2"}
|
| 250 |
+
{"event": "train", "step": 5800, "total_steps": 6315, "epoch_fraction": 2.7555, "loss": 0.9052460789680481, "classification_loss": 0.08203292638063431, "lm_loss": 1.6464266777038574, "lr": 5.117423040304665e-06, "updates_per_second": 6.125, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 944.6, "task": "sst2"}
|
| 251 |
+
{"event": "train", "step": 5825, "total_steps": 6315, "epoch_fraction": 2.7673, "loss": 0.8657037019729614, "classification_loss": 0.03390316665172577, "lm_loss": 1.6636009216308594, "lr": 4.8694858774992065e-06, "updates_per_second": 6.36, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 948.5, "task": "sst2"}
|
| 252 |
+
{"event": "train", "step": 5850, "total_steps": 6315, "epoch_fraction": 2.7792, "loss": 0.8665140867233276, "classification_loss": 0.04780648648738861, "lm_loss": 1.6374154090881348, "lr": 4.621548714693748e-06, "updates_per_second": 6.284, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 952.5, "task": "sst2"}
|
| 253 |
+
{"event": "train", "step": 5875, "total_steps": 6315, "epoch_fraction": 2.7911, "loss": 0.8701431751251221, "classification_loss": 0.03749920800328255, "lm_loss": 1.6652876138687134, "lr": 4.37361155188829e-06, "updates_per_second": 6.301, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 956.5, "task": "sst2"}
|
| 254 |
+
{"event": "train", "step": 5900, "total_steps": 6315, "epoch_fraction": 2.803, "loss": 0.8809349536895752, "classification_loss": 0.0801820233464241, "lm_loss": 1.6015058755874634, "lr": 4.125674389082832e-06, "updates_per_second": 6.263, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 960.5, "task": "sst2"}
|
| 255 |
+
{"event": "train", "step": 5925, "total_steps": 6315, "epoch_fraction": 2.8149, "loss": 0.8848820924758911, "classification_loss": 0.05972805991768837, "lm_loss": 1.65030837059021, "lr": 3.877737226277372e-06, "updates_per_second": 6.319, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 964.4, "task": "sst2"}
|
| 256 |
+
{"event": "train", "step": 5950, "total_steps": 6315, "epoch_fraction": 2.8267, "loss": 0.825607419013977, "classification_loss": 0.04800703376531601, "lm_loss": 1.5552009344100952, "lr": 3.6298000634719137e-06, "updates_per_second": 6.355, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 968.4, "task": "sst2"}
|
| 257 |
+
{"event": "train", "step": 5975, "total_steps": 6315, "epoch_fraction": 2.8386, "loss": 0.876387894153595, "classification_loss": 0.05143048241734505, "lm_loss": 1.6499149799346924, "lr": 3.3818629006664555e-06, "updates_per_second": 6.365, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 972.3, "task": "sst2"}
|
| 258 |
+
{"event": "train", "step": 6000, "total_steps": 6315, "epoch_fraction": 2.8505, "loss": 0.8284080028533936, "classification_loss": 0.04222726821899414, "lm_loss": 1.5723614692687988, "lr": 3.1339257378609965e-06, "updates_per_second": 6.403, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 976.2, "task": "sst2"}
|
| 259 |
+
{"event": "eval", "step": 6000, "epoch": 2, "offset": 57280, "train": {"correct": 127, "total": 128, "accuracy": 0.9921875, "loss": 0.021897166967391968, "gold_label_counts": [65, 63], "majority_accuracy": 0.5078125}, "validation": {"correct": 782, "total": 872, "accuracy": 0.8967889908256881, "loss": 0.3115563988685608, "gold_label_counts": [428, 444], "majority_accuracy": 0.5091743119266054}, "best_step": 1000, "omega_std": 0.6023039221763611, "peak_allocated_gib": 3.846, "sample": {"gold": 1, "predicted": 1}, "elapsed_seconds": 977.1, "task": "sst2"}
|
| 260 |
+
{"event": "train", "step": 6025, "total_steps": 6315, "epoch_fraction": 2.8624, "loss": 0.8710504174232483, "classification_loss": 0.05635256692767143, "lm_loss": 1.6293957233428955, "lr": 2.885988575055538e-06, "updates_per_second": 6.251, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 982.3, "task": "sst2"}
|
| 261 |
+
{"event": "train", "step": 6050, "total_steps": 6315, "epoch_fraction": 2.8743, "loss": 0.8888440728187561, "classification_loss": 0.043443307280540466, "lm_loss": 1.6908013820648193, "lr": 2.6380514122500794e-06, "updates_per_second": 6.288, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 986.3, "task": "sst2"}
|
| 262 |
+
{"event": "train", "step": 6075, "total_steps": 6315, "epoch_fraction": 2.8861, "loss": 0.8692652583122253, "classification_loss": 0.054785337299108505, "lm_loss": 1.6289597749710083, "lr": 2.390114249444621e-06, "updates_per_second": 6.194, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 990.3, "task": "sst2"}
|
| 263 |
+
{"event": "train", "step": 6100, "total_steps": 6315, "epoch_fraction": 2.898, "loss": 0.8494734764099121, "classification_loss": 0.04687304422259331, "lm_loss": 1.6052008867263794, "lr": 2.1421770866391623e-06, "updates_per_second": 6.208, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 994.4, "task": "sst2"}
|
| 264 |
+
{"event": "train", "step": 6125, "total_steps": 6315, "epoch_fraction": 2.9099, "loss": 0.8558051586151123, "classification_loss": 0.04959813877940178, "lm_loss": 1.6124138832092285, "lr": 1.8942399238337037e-06, "updates_per_second": 6.117, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 998.5, "task": "sst2"}
|
| 265 |
+
{"event": "train", "step": 6150, "total_steps": 6315, "epoch_fraction": 2.9218, "loss": 0.8593693375587463, "classification_loss": 0.05009318143129349, "lm_loss": 1.6185522079467773, "lr": 1.6463027610282451e-06, "updates_per_second": 6.384, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 1002.4, "task": "sst2"}
|
| 266 |
+
{"event": "train", "step": 6175, "total_steps": 6315, "epoch_fraction": 2.9336, "loss": 0.8850686550140381, "classification_loss": 0.08319608867168427, "lm_loss": 1.6037451028823853, "lr": 1.3983655982227864e-06, "updates_per_second": 6.453, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 1006.3, "task": "sst2"}
|
| 267 |
+
{"event": "train", "step": 6200, "total_steps": 6315, "epoch_fraction": 2.9455, "loss": 0.847789466381073, "classification_loss": 0.037783920764923096, "lm_loss": 1.6200109720230103, "lr": 1.1504284354173278e-06, "updates_per_second": 6.455, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 1010.1, "task": "sst2"}
|
| 268 |
+
{"event": "train", "step": 6225, "total_steps": 6315, "epoch_fraction": 2.9574, "loss": 0.8368501663208008, "classification_loss": 0.034198563545942307, "lm_loss": 1.605303168296814, "lr": 9.024912726118692e-07, "updates_per_second": 6.494, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 1014.0, "task": "sst2"}
|
| 269 |
+
{"event": "train", "step": 6250, "total_steps": 6315, "epoch_fraction": 2.9693, "loss": 0.8687301278114319, "classification_loss": 0.0437571182847023, "lm_loss": 1.6499459743499756, "lr": 6.545541098064108e-07, "updates_per_second": 6.493, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 1017.8, "task": "sst2"}
|
| 270 |
+
{"event": "train", "step": 6275, "total_steps": 6315, "epoch_fraction": 2.9812, "loss": 0.8756495714187622, "classification_loss": 0.08067388087511063, "lm_loss": 1.5899512767791748, "lr": 4.0661694700095213e-07, "updates_per_second": 6.47, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 1021.7, "task": "sst2"}
|
| 271 |
+
{"event": "train", "step": 6300, "total_steps": 6315, "epoch_fraction": 2.993, "loss": 0.8401745557785034, "classification_loss": 0.047997813671827316, "lm_loss": 1.5843533277511597, "lr": 1.5867978419549348e-07, "updates_per_second": 6.344, "peak_allocated_gib": 3.846, "nonfinite_updates": 0, "elapsed_seconds": 1025.6, "task": "sst2"}
|
| 272 |
+
{"event": "eval", "step": 6315, "epoch": 3, "offset": 0, "train": {"correct": 126, "total": 128, "accuracy": 0.984375, "loss": 0.020622102543711662, "gold_label_counts": [65, 63], "majority_accuracy": 0.5078125}, "validation": {"correct": 786, "total": 872, "accuracy": 0.9013761467889908, "loss": 0.29649588465690613, "gold_label_counts": [428, 444], "majority_accuracy": 0.5091743119266054}, "best_step": 6315, "omega_std": 0.6023041605949402, "peak_allocated_gib": 3.846, "sample": {"gold": 1, "predicted": 1}, "elapsed_seconds": 1028.8, "task": "sst2"}
|
| 273 |
+
{"event": "complete", "step": 6315, "best_step": 6315, "stopping_cause": "epochs_complete", "heldout": "validation", "before": {"correct": 437, "total": 872, "accuracy": 0.5011467889908257, "loss": 4.14792537689209, "gold_label_counts": [428, 444], "majority_accuracy": 0.5091743119266054}, "after": {"correct": 786, "total": 872, "accuracy": 0.9013761467889908, "loss": 0.29649588465690613, "gold_label_counts": [428, 444], "majority_accuracy": 0.5091743119266054}, "paired_probe": {"before_correct": 50, "after_correct": 92, "total": 100}, "elapsed_seconds": 1032.6, "task": "sst2"}
|
task_finetuning/results/sst2_provenance.json
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_source_sha256": "a28201b8194941a75d3ca2e0812e6d2d0eef1b2656cc0c9f8fd0ca65bf96b28b",
|
| 3 |
+
"trainer_sha256": "550f68528e3dbe630cb7f0ad29860d8bafb77767832819dab86ea598a29ad520",
|
| 4 |
+
"starting_checkpoint": {
|
| 5 |
+
"checkpoint": "original_workspace/Main folders/WiggleGPT v3/WiggleGPT/out-wigglegpt-pure-124m/ckpt.pt",
|
| 6 |
+
"saved_step": 600000,
|
| 7 |
+
"parameters": 123697920,
|
| 8 |
+
"model_args": {
|
| 9 |
+
"n_layer": 12,
|
| 10 |
+
"n_head": 12,
|
| 11 |
+
"n_embd": 768,
|
| 12 |
+
"block_size": 1024,
|
| 13 |
+
"bias": false,
|
| 14 |
+
"vocab_size": 50304,
|
| 15 |
+
"dropout": 0.0,
|
| 16 |
+
"use_bio_mlp": true,
|
| 17 |
+
"use_rmsnorm": true,
|
| 18 |
+
"use_rope": true,
|
| 19 |
+
"gradient_checkpointing": false
|
| 20 |
+
},
|
| 21 |
+
"torch": "2.11.0.dev20260215+cu128",
|
| 22 |
+
"checkpoint_sha256": "88afaab6c93ef3e33940b4e25cbce2f44038c428109600afe8285586a3a767b9"
|
| 23 |
+
},
|
| 24 |
+
"torch": "2.11.0.dev20260215+cu128",
|
| 25 |
+
"markers": {
|
| 26 |
+
"start": 50257,
|
| 27 |
+
"delimiter": 50258,
|
| 28 |
+
"classify": 50259,
|
| 29 |
+
"pad": 50256
|
| 30 |
+
},
|
| 31 |
+
"native_backbone_dropout": 0.0,
|
| 32 |
+
"data": {
|
| 33 |
+
"train": {
|
| 34 |
+
"example_ids_summary": {
|
| 35 |
+
"count": 67349,
|
| 36 |
+
"sha256_json": "2407adf1dd74958d0bc7a0d05034729d11b0cd55a1e3a357e44e0b2e6e3701b1"
|
| 37 |
+
},
|
| 38 |
+
"metadata": {
|
| 39 |
+
"task": "sst2",
|
| 40 |
+
"split": "train",
|
| 41 |
+
"examples": 67349,
|
| 42 |
+
"choices": 1,
|
| 43 |
+
"source_jsonl": "original_workspace/output/gpt1_style/task_finetuning/data/sst2_train.jsonl",
|
| 44 |
+
"source_sha256": "75fd14e7a62a5d54d1300996afaef6223ec8c49eecd97f0674d6664489a69daa",
|
| 45 |
+
"pad_length": 80,
|
| 46 |
+
"max_sequence_length": 67,
|
| 47 |
+
"min_sequence_length": 4,
|
| 48 |
+
"max_untruncated_length": 67,
|
| 49 |
+
"truncation": {
|
| 50 |
+
"examples_with_truncation": 0,
|
| 51 |
+
"candidate_sequences_with_truncation": 0,
|
| 52 |
+
"total_tokens_removed": 0,
|
| 53 |
+
"max_tokens_removed_per_example": 0
|
| 54 |
+
},
|
| 55 |
+
"context_policy": "[START] + review first 510 tokens + [CLS]; right-pad to the split maximum rounded up to a multiple of 16 (minimum 16).",
|
| 56 |
+
"input_ids_dtype": "int32",
|
| 57 |
+
"lengths_dtype": "int64",
|
| 58 |
+
"labels_dtype": "int64"
|
| 59 |
+
},
|
| 60 |
+
"source_sha256": "75fd14e7a62a5d54d1300996afaef6223ec8c49eecd97f0674d6664489a69daa"
|
| 61 |
+
},
|
| 62 |
+
"validation": {
|
| 63 |
+
"example_ids_summary": {
|
| 64 |
+
"count": 872,
|
| 65 |
+
"sha256_json": "5c2ec0e8e6c25bfe98bfa1769994fb1d2b1ac092caab6423f798d5a63cb01c18"
|
| 66 |
+
},
|
| 67 |
+
"metadata": {
|
| 68 |
+
"task": "sst2",
|
| 69 |
+
"split": "validation",
|
| 70 |
+
"examples": 872,
|
| 71 |
+
"choices": 1,
|
| 72 |
+
"source_jsonl": "original_workspace/output/gpt1_style/task_finetuning/data/sst2_validation.jsonl",
|
| 73 |
+
"source_sha256": "9f7afa11e8d0309f4ea4732c7f11d78fb0f686f48fac5596a06db2ecf496678b",
|
| 74 |
+
"pad_length": 64,
|
| 75 |
+
"max_sequence_length": 63,
|
| 76 |
+
"min_sequence_length": 5,
|
| 77 |
+
"max_untruncated_length": 63,
|
| 78 |
+
"truncation": {
|
| 79 |
+
"examples_with_truncation": 0,
|
| 80 |
+
"candidate_sequences_with_truncation": 0,
|
| 81 |
+
"total_tokens_removed": 0,
|
| 82 |
+
"max_tokens_removed_per_example": 0
|
| 83 |
+
},
|
| 84 |
+
"context_policy": "[START] + review first 510 tokens + [CLS]; right-pad to the split maximum rounded up to a multiple of 16 (minimum 16).",
|
| 85 |
+
"input_ids_dtype": "int32",
|
| 86 |
+
"lengths_dtype": "int64",
|
| 87 |
+
"labels_dtype": "int64"
|
| 88 |
+
},
|
| 89 |
+
"source_sha256": "9f7afa11e8d0309f4ea4732c7f11d78fb0f686f48fac5596a06db2ecf496678b"
|
| 90 |
+
}
|
| 91 |
+
},
|
| 92 |
+
"all_backbone_parameters_train": true
|
| 93 |
+
}
|
task_finetuning/results/train_tasks_original_source.txt
ADDED
|
@@ -0,0 +1,449 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Full-model GPT-1-style task fine-tuning of original WiggleGPT.
|
| 2 |
+
|
| 3 |
+
Native model_bio.py is imported unchanged. A final-state classifier is added;
|
| 4 |
+
its language-model objective uses the native tied projection. No CPU forwards.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import argparse
|
| 9 |
+
from collections import Counter
|
| 10 |
+
from datetime import datetime
|
| 11 |
+
from zoneinfo import ZoneInfo
|
| 12 |
+
import gc
|
| 13 |
+
import hashlib
|
| 14 |
+
import json
|
| 15 |
+
import math
|
| 16 |
+
import os
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
import random
|
| 19 |
+
import shutil
|
| 20 |
+
import signal
|
| 21 |
+
import sys
|
| 22 |
+
import time
|
| 23 |
+
|
| 24 |
+
ROOT = Path(__file__).resolve().parent
|
| 25 |
+
WORKSPACE = ROOT.parents[2]
|
| 26 |
+
NATIVE = WORKSPACE / 'Main folders' / 'WiggleGPT v3' / 'WiggleGPT'
|
| 27 |
+
sys.path.insert(0, str(WORKSPACE / 'WiggleGPT-Gosport'))
|
| 28 |
+
sys.path.insert(0, str(NATIVE))
|
| 29 |
+
from wiggle_assistant import load_model
|
| 30 |
+
import model_bio
|
| 31 |
+
import torch
|
| 32 |
+
from torch import nn
|
| 33 |
+
from torch.nn import functional as F
|
| 34 |
+
|
| 35 |
+
STOP = False
|
| 36 |
+
MARKERS = {'start': 50257, 'delimiter': 50258, 'classify': 50259, 'pad': 50256}
|
| 37 |
+
|
| 38 |
+
def stop_requested(signum, frame):
|
| 39 |
+
global STOP
|
| 40 |
+
STOP = True
|
| 41 |
+
print('Stop requested; finishing the update and saving latest.pt.', flush=True)
|
| 42 |
+
|
| 43 |
+
def digest(path):
|
| 44 |
+
with Path(path).open('rb') as stream:
|
| 45 |
+
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
| 46 |
+
|
| 47 |
+
class TaskModel(nn.Module):
|
| 48 |
+
def __init__(self, base, task, dropout):
|
| 49 |
+
super().__init__()
|
| 50 |
+
self.base = base
|
| 51 |
+
self.task = task
|
| 52 |
+
self.dropout = dropout
|
| 53 |
+
# Added linear classifier is the requested GPT-1 task head, not a backbone replacement.
|
| 54 |
+
with torch.device('cuda:0'):
|
| 55 |
+
self.classifier = nn.Linear(base.config.n_embd, 2 if task == 'sst2' else 1)
|
| 56 |
+
nn.init.normal_(self.classifier.weight, mean=0.0, std=0.02)
|
| 57 |
+
nn.init.zeros_(self.classifier.bias)
|
| 58 |
+
self.hidden = None
|
| 59 |
+
self.base.transformer.ln_f.register_forward_hook(self.capture_hidden)
|
| 60 |
+
|
| 61 |
+
def capture_hidden(self, module, args, output):
|
| 62 |
+
self.hidden = output
|
| 63 |
+
|
| 64 |
+
def forward(self, tokens, lengths, auxiliary=False):
|
| 65 |
+
batch, choices, width = tokens.shape
|
| 66 |
+
flat = tokens.reshape(batch * choices, width)
|
| 67 |
+
self.base(flat)
|
| 68 |
+
hidden = self.hidden
|
| 69 |
+
self.hidden = None
|
| 70 |
+
positions = lengths.reshape(-1) - 1
|
| 71 |
+
features = hidden[torch.arange(batch * choices, device=flat.device), positions]
|
| 72 |
+
features = features.reshape(batch, choices, -1)
|
| 73 |
+
# GPT-1's multiple-choice head shares a dropout feature mask across options.
|
| 74 |
+
if self.training and self.dropout:
|
| 75 |
+
mask = F.dropout(torch.ones((batch, 1, features.shape[-1]), device=features.device,
|
| 76 |
+
dtype=features.dtype), p=self.dropout, training=True)
|
| 77 |
+
features = features * mask
|
| 78 |
+
scores = self.classifier(features).squeeze(1) if self.task == 'sst2' else self.classifier(features).squeeze(-1)
|
| 79 |
+
lm_loss = None
|
| 80 |
+
if auxiliary:
|
| 81 |
+
logits = self.base.lm_head(hidden[:, :-1])
|
| 82 |
+
targets = flat[:, 1:].long()
|
| 83 |
+
per_token = F.cross_entropy(logits.float().reshape(-1, logits.shape[-1]),
|
| 84 |
+
targets.reshape(-1), reduction='none').view_as(targets)
|
| 85 |
+
mask = torch.arange(width - 1, device=flat.device)[None, :] < (lengths.reshape(-1, 1) - 1)
|
| 86 |
+
# Average within each candidate sequence, then across all candidates.
|
| 87 |
+
lm_loss = ((per_token * mask).sum(-1) / mask.sum(-1).clamp_min(1)).mean()
|
| 88 |
+
return scores.float(), lm_loss
|
| 89 |
+
|
| 90 |
+
def load_data(task, device):
|
| 91 |
+
prefix = 'sst2' if task == 'sst2' else 'race_middle'
|
| 92 |
+
splits = ['train', 'validation'] + (['test'] if task != 'sst2' else [])
|
| 93 |
+
data = {}
|
| 94 |
+
for split in splits:
|
| 95 |
+
path = ROOT / 'data' / f'{prefix}_{split}_encoded.pt'
|
| 96 |
+
if not path.exists():
|
| 97 |
+
# Preparation script may use the compact .pt suffix.
|
| 98 |
+
path = ROOT / 'data' / f'{prefix}_{split}.pt'
|
| 99 |
+
cache = torch.load(path, map_location='cpu', weights_only=False, mmap=True)
|
| 100 |
+
# Both conventional names are accepted; contents stay unchanged.
|
| 101 |
+
ids = cache.get('input_ids', cache.get('tokens'))
|
| 102 |
+
data[split] = {'input_ids': ids.to(device), 'lengths': cache['lengths'].to(device),
|
| 103 |
+
'labels': cache['labels'].to(device), 'cache_path': str(path),
|
| 104 |
+
'metadata': {k:v for k,v in cache.items() if not torch.is_tensor(v)}}
|
| 105 |
+
return data
|
| 106 |
+
|
| 107 |
+
def select_batch(data, index):
|
| 108 |
+
return data['input_ids'][index], data['lengths'][index], data['labels'][index]
|
| 109 |
+
|
| 110 |
+
def evaluate(model, data, batch_size, fixed_indices=None):
|
| 111 |
+
model.eval()
|
| 112 |
+
predictions, losses = [], []
|
| 113 |
+
if fixed_indices is None:
|
| 114 |
+
fixed_indices = torch.arange(data['labels'].shape[0], device='cuda:0')
|
| 115 |
+
# AMP-COMPLEX: native weights/activations are real; native RoPE explicitly casts its complex rotation to float32.
|
| 116 |
+
with torch.inference_mode(), torch.autocast('cuda', dtype=torch.bfloat16):
|
| 117 |
+
for start in range(0, len(fixed_indices), batch_size):
|
| 118 |
+
indices = fixed_indices[start:start + batch_size]
|
| 119 |
+
x, lengths, labels = select_batch(data, indices)
|
| 120 |
+
logits, _ = model(x, lengths)
|
| 121 |
+
predictions.append(logits.argmax(-1))
|
| 122 |
+
losses.append(F.cross_entropy(logits, labels, reduction='sum'))
|
| 123 |
+
predicted = torch.cat(predictions)
|
| 124 |
+
labels = data['labels'][fixed_indices]
|
| 125 |
+
correct = int((predicted == labels).sum())
|
| 126 |
+
counts = torch.bincount(labels, minlength=2 if model.task == 'sst2' else 4).cpu().tolist()
|
| 127 |
+
return {'correct': correct, 'total': len(labels), 'accuracy': correct / len(labels),
|
| 128 |
+
'loss': float(torch.stack(losses).sum() / len(labels)),
|
| 129 |
+
'predictions': predicted.cpu().tolist(), 'gold_label_counts': counts,
|
| 130 |
+
'majority_accuracy': max(counts) / len(labels)}
|
| 131 |
+
|
| 132 |
+
def optimizer_for(model, config):
|
| 133 |
+
decay, no_decay = [], []
|
| 134 |
+
protected = ('omega', 'phi', 'baseline', 'norm', 'bias', 'scale', 'phase')
|
| 135 |
+
for name, parameter in model.named_parameters():
|
| 136 |
+
if parameter.ndim >= 2 and not parameter.is_complex() and not any(word in name for word in protected):
|
| 137 |
+
decay.append(parameter)
|
| 138 |
+
else:
|
| 139 |
+
no_decay.append(parameter)
|
| 140 |
+
print(f'Decay: {sum(p.numel() for p in decay):,}; protected/no decay: {sum(p.numel() for p in no_decay):,}', flush=True)
|
| 141 |
+
return torch.optim.AdamW([{'params':decay, 'weight_decay':config['weight_decay']},
|
| 142 |
+
{'params':no_decay, 'weight_decay':0.0}],
|
| 143 |
+
lr=config['learning_rate'], betas=tuple(config['betas']), eps=config['epsilon'], foreach=True)
|
| 144 |
+
|
| 145 |
+
def schedule(step, total, config):
|
| 146 |
+
warmup = max(1, round(total * config['warmup_fraction']))
|
| 147 |
+
if step < warmup:
|
| 148 |
+
return config['learning_rate'] * (step + 1) / warmup
|
| 149 |
+
return config['learning_rate'] * max(0.0, (total - step) / max(1, total - warmup))
|
| 150 |
+
|
| 151 |
+
def atomic_save(path, payload):
|
| 152 |
+
temporary = path.with_suffix('.pending.pt')
|
| 153 |
+
torch.save(payload, temporary)
|
| 154 |
+
temporary.replace(path)
|
| 155 |
+
|
| 156 |
+
def rng_state(generator):
|
| 157 |
+
return {'python': random.getstate(), 'torch': torch.get_rng_state(),
|
| 158 |
+
'cuda': torch.cuda.get_rng_state(0), 'batch_generator': generator.get_state()}
|
| 159 |
+
|
| 160 |
+
def restore_rng(state, generator):
|
| 161 |
+
random.setstate(state['python'])
|
| 162 |
+
torch.set_rng_state(state['torch'].cpu())
|
| 163 |
+
torch.cuda.set_rng_state(state['cuda'].cpu(), 0)
|
| 164 |
+
generator.set_state(state['batch_generator'].cpu())
|
| 165 |
+
|
| 166 |
+
def full_summary(result):
|
| 167 |
+
return {k:v for k,v in result.items() if k != 'predictions'}
|
| 168 |
+
|
| 169 |
+
def raw_rows(task, split):
|
| 170 |
+
path = ROOT / 'data' / f'{task}_{split}.jsonl'
|
| 171 |
+
return [json.loads(line) for line in path.read_text(encoding='utf-8').splitlines()]
|
| 172 |
+
|
| 173 |
+
def run_task(task, config, resume=None):
|
| 174 |
+
task_config = config['tasks'][task]
|
| 175 |
+
random.seed(config['seed'])
|
| 176 |
+
torch.manual_seed(config['seed'])
|
| 177 |
+
torch.cuda.manual_seed_all(config['seed'])
|
| 178 |
+
base, native_config, encoder, metadata = load_model((ROOT / config['starting_checkpoint']).resolve())
|
| 179 |
+
free, total = torch.cuda.mem_get_info()
|
| 180 |
+
cap = min(int(config['max_process_gib'] * 1024**3),
|
| 181 |
+
total - int(config['desktop_reserve_gib'] * 1024**3),
|
| 182 |
+
free + torch.cuda.memory_reserved() - int(0.5 * 1024**3))
|
| 183 |
+
torch.cuda.set_per_process_memory_fraction(cap / total, 0)
|
| 184 |
+
base.config.gradient_checkpointing = task_config['gradient_checkpointing']
|
| 185 |
+
metadata['checkpoint_sha256'] = digest((ROOT / config['starting_checkpoint']).resolve())
|
| 186 |
+
model = TaskModel(base, task, config['classifier_dropout'])
|
| 187 |
+
# Reuse three existing padding slots; real GPT-2 IDs and tokenizer are unchanged.
|
| 188 |
+
with torch.no_grad():
|
| 189 |
+
marker_ids = torch.tensor([MARKERS['start'],MARKERS['delimiter'],MARKERS['classify']], device='cuda:0')
|
| 190 |
+
base.transformer.wte.weight.index_copy_(0, marker_ids,
|
| 191 |
+
torch.randn((3, base.config.n_embd), device='cuda:0') * 0.02)
|
| 192 |
+
optimizer = optimizer_for(model, config)
|
| 193 |
+
data = load_data(task, 'cuda:0')
|
| 194 |
+
n_train = data['train']['labels'].shape[0]
|
| 195 |
+
per_epoch = math.ceil(n_train / config['effective_batch_size'])
|
| 196 |
+
total_steps = per_epoch * config['epochs']
|
| 197 |
+
generator = torch.Generator(device='cuda:0').manual_seed(config['seed'] + 123)
|
| 198 |
+
if resume:
|
| 199 |
+
checkpoint = torch.load(resume, map_location='cpu', weights_only=False, mmap=True)
|
| 200 |
+
model.load_state_dict(checkpoint['model'], strict=True)
|
| 201 |
+
optimizer.load_state_dict(checkpoint['optimizer'])
|
| 202 |
+
state = checkpoint['state']
|
| 203 |
+
order = checkpoint['order'].to('cuda:0') if checkpoint['order'] is not None else None
|
| 204 |
+
restore_rng(checkpoint['rng'], generator)
|
| 205 |
+
run_dir = Path(resume).resolve().parent
|
| 206 |
+
del checkpoint
|
| 207 |
+
else:
|
| 208 |
+
run_dir = ROOT / 'runs' / f'train_tasks_{datetime.now(ZoneInfo("Europe/London")).strftime("%Y-%m-%d_%H-%M-%S")}_{task}'
|
| 209 |
+
run_dir.mkdir(parents=True, exist_ok=False)
|
| 210 |
+
state = {'step':0, 'epoch':0, 'offset':0, 'best_accuracy':-1.0, 'best_loss':float('inf'),
|
| 211 |
+
'best_step':0, 'nonfinite_updates':0, 'consecutive_nonfinite':0, 'last_eval_step':-1,
|
| 212 |
+
'elapsed_seconds':0.0, 'lr_scale':1.0}
|
| 213 |
+
order = None
|
| 214 |
+
(run_dir / 'config.json').write_text(json.dumps(config, indent=2), encoding='utf-8')
|
| 215 |
+
shutil.copy2(__file__, run_dir / 'train_tasks.py')
|
| 216 |
+
shutil.copy2(model_bio.__file__, run_dir / 'model_bio.py')
|
| 217 |
+
provenance = {'model_source_sha256':digest(model_bio.__file__), 'trainer_sha256':digest(__file__),
|
| 218 |
+
'starting_checkpoint':metadata, 'torch':torch.__version__, 'markers':MARKERS,
|
| 219 |
+
'native_backbone_dropout':base.config.dropout,
|
| 220 |
+
'data':{split:d['metadata'] for split,d in data.items()},
|
| 221 |
+
'all_backbone_parameters_train':True}
|
| 222 |
+
(run_dir/'provenance.json').write_text(json.dumps(provenance, indent=2), encoding='utf-8')
|
| 223 |
+
log = (run_dir / (run_dir.name + '.log')).open('a', encoding='utf-8', buffering=1)
|
| 224 |
+
def emit(event):
|
| 225 |
+
event.update(task=task)
|
| 226 |
+
line = json.dumps(event, ensure_ascii=False)
|
| 227 |
+
print(line, flush=True)
|
| 228 |
+
log.write(line + '\n')
|
| 229 |
+
with (run_dir/'metrics.jsonl').open('a',encoding='utf-8') as stream:
|
| 230 |
+
stream.write(line + '\n')
|
| 231 |
+
def save(name):
|
| 232 |
+
atomic_save(run_dir/name, {'model':model.state_dict(), 'optimizer':optimizer.state_dict(),
|
| 233 |
+
'state':dict(state), 'order':order, 'rng':rng_state(generator), 'config':config,
|
| 234 |
+
'task':task, 'model_args':vars(base.config), 'markers':MARKERS,
|
| 235 |
+
'source_hashes':{'model':digest(model_bio.__file__),'trainer':digest(__file__)}})
|
| 236 |
+
evaluation_batch = 16 if task == 'sst2' else 4
|
| 237 |
+
heldout = 'validation' if task == 'sst2' else 'test'
|
| 238 |
+
before_path = run_dir/'before.json'
|
| 239 |
+
start_time = time.perf_counter()
|
| 240 |
+
elapsed_prior = state['elapsed_seconds']
|
| 241 |
+
emit({'event':'start','step':state['step'],'train_examples':n_train,'epochs':config['epochs'],
|
| 242 |
+
'total_steps':total_steps,'microbatch':task_config['microbatch_size'],
|
| 243 |
+
'effective_batch':config['effective_batch_size'],'memory_cap_gib':round(cap/1024**3,2),
|
| 244 |
+
'trainable_parameters':sum(p.numel() for p in model.parameters() if p.requires_grad),
|
| 245 |
+
'device':torch.cuda.get_device_name(0)})
|
| 246 |
+
if not before_path.exists():
|
| 247 |
+
before = evaluate(model, data[heldout], evaluation_batch)
|
| 248 |
+
before_path.write_text(json.dumps(before,indent=2),encoding='utf-8')
|
| 249 |
+
emit({'event':'before','heldout':heldout,**full_summary(before)})
|
| 250 |
+
else:
|
| 251 |
+
before = json.loads(before_path.read_text(encoding='utf-8'))
|
| 252 |
+
train_fixed = torch.arange(min(128,n_train), device='cuda:0')
|
| 253 |
+
def evaluate_and_save():
|
| 254 |
+
validation = evaluate(model, data['validation'], evaluation_batch)
|
| 255 |
+
train_eval = evaluate(model, data['train'], evaluation_batch, train_fixed)
|
| 256 |
+
changed = (validation['accuracy'] > state['best_accuracy'] or
|
| 257 |
+
(validation['accuracy'] == state['best_accuracy'] and validation['loss'] < state['best_loss']))
|
| 258 |
+
if changed:
|
| 259 |
+
state['best_accuracy'] = validation['accuracy']
|
| 260 |
+
state['best_loss'] = validation['loss']
|
| 261 |
+
state['best_step'] = state['step']
|
| 262 |
+
state['last_eval_step'] = state['step']
|
| 263 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter() - start_time
|
| 264 |
+
# Once-per-evaluation diagnostic over twelve oscillator tensors, not Python-built training batches.
|
| 265 |
+
omegas = torch.cat([p.detach().reshape(-1) for name,p in model.named_parameters() if name.endswith('.omega')])
|
| 266 |
+
emit({'event':'eval','step':state['step'],'epoch':state['epoch'],'offset':state['offset'],
|
| 267 |
+
'train':full_summary(train_eval),'validation':full_summary(validation),
|
| 268 |
+
'best_step':state['best_step'],'omega_std':float(omegas.std(correction=0)),
|
| 269 |
+
'peak_allocated_gib':round(torch.cuda.max_memory_allocated()/1024**3,3),
|
| 270 |
+
'sample':{'gold':int(data['validation']['labels'][0]),'predicted':validation['predictions'][0]},
|
| 271 |
+
'elapsed_seconds':round(state['elapsed_seconds'],1)})
|
| 272 |
+
if changed:
|
| 273 |
+
save('best.pt')
|
| 274 |
+
save('latest.pt')
|
| 275 |
+
model.train()
|
| 276 |
+
if state['last_eval_step'] < 0:
|
| 277 |
+
evaluate_and_save()
|
| 278 |
+
model.train()
|
| 279 |
+
accumulated = torch.zeros(3,device='cuda:0')
|
| 280 |
+
logged_steps = 0
|
| 281 |
+
log_start = time.perf_counter()
|
| 282 |
+
stopping_cause = 'epochs_complete'
|
| 283 |
+
while state['epoch'] < config['epochs']:
|
| 284 |
+
if order is None:
|
| 285 |
+
order = torch.randperm(n_train,device='cuda:0',generator=generator)
|
| 286 |
+
while state['offset'] < n_train:
|
| 287 |
+
end = min(n_train,state['offset']+config['effective_batch_size'])
|
| 288 |
+
indices = order[state['offset']:end]
|
| 289 |
+
effective = len(indices)
|
| 290 |
+
lr = schedule(state['step'], total_steps, config) * state['lr_scale']
|
| 291 |
+
for group in optimizer.param_groups:
|
| 292 |
+
group['lr'] = lr
|
| 293 |
+
optimizer.zero_grad(set_to_none=True)
|
| 294 |
+
batch_values = torch.zeros(3,device='cuda:0')
|
| 295 |
+
for position in range(0,effective,task_config['microbatch_size']):
|
| 296 |
+
micro = indices[position:position+task_config['microbatch_size']]
|
| 297 |
+
x,lengths,labels = select_batch(data['train'],micro)
|
| 298 |
+
# AMP-COMPLEX: real-valued model; native rotary complex operations remain explicitly float32.
|
| 299 |
+
with torch.autocast('cuda',dtype=torch.bfloat16):
|
| 300 |
+
logits,lm_loss = model(x,lengths,auxiliary=True)
|
| 301 |
+
classification_loss = F.cross_entropy(logits,labels)
|
| 302 |
+
loss = classification_loss + config['lm_coefficient'] * lm_loss
|
| 303 |
+
weighted = loss * (len(micro)/effective)
|
| 304 |
+
weighted.backward()
|
| 305 |
+
batch_values += torch.stack((loss.detach(),classification_loss.detach(),lm_loss.detach())) * (len(micro)/effective)
|
| 306 |
+
norm = torch.nn.utils.clip_grad_norm_(model.parameters(),config['grad_clip'])
|
| 307 |
+
if torch.isfinite(norm):
|
| 308 |
+
optimizer.step()
|
| 309 |
+
state['consecutive_nonfinite'] = 0
|
| 310 |
+
else:
|
| 311 |
+
state['nonfinite_updates'] += 1
|
| 312 |
+
state['consecutive_nonfinite'] += 1
|
| 313 |
+
if state['consecutive_nonfinite'] >= 2:
|
| 314 |
+
state['lr_scale'] *= 0.5
|
| 315 |
+
emit({'event':'nonfinite_update_skipped','step':state['step']+1,
|
| 316 |
+
'count':state['nonfinite_updates'],'lr_scale':state['lr_scale']})
|
| 317 |
+
state['step'] += 1
|
| 318 |
+
state['offset'] = end
|
| 319 |
+
accumulated += batch_values
|
| 320 |
+
logged_steps += 1
|
| 321 |
+
if state['step'] == 1 or state['step'] % config['log_interval'] == 0:
|
| 322 |
+
# One host transfer per logging interval; hot microbatches remain on CUDA.
|
| 323 |
+
values = (accumulated/logged_steps).cpu().tolist()
|
| 324 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter()-start_time
|
| 325 |
+
emit({'event':'train','step':state['step'],'total_steps':total_steps,
|
| 326 |
+
'epoch_fraction':round(state['epoch']+end/n_train,4),'loss':values[0],
|
| 327 |
+
'classification_loss':values[1],'lm_loss':values[2],'lr':lr,
|
| 328 |
+
'updates_per_second':round(logged_steps/(time.perf_counter()-log_start),3),
|
| 329 |
+
'peak_allocated_gib':round(torch.cuda.max_memory_allocated()/1024**3,3),
|
| 330 |
+
'nonfinite_updates':state['nonfinite_updates'],'elapsed_seconds':round(state['elapsed_seconds'],1)})
|
| 331 |
+
accumulated.zero_(); logged_steps=0;log_start=time.perf_counter()
|
| 332 |
+
if state['step'] % task_config['eval_interval'] == 0:
|
| 333 |
+
evaluate_and_save()
|
| 334 |
+
log_start=time.perf_counter()
|
| 335 |
+
elif state['step'] % config['checkpoint_interval'] == 0:
|
| 336 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter()-start_time
|
| 337 |
+
save('latest.pt')
|
| 338 |
+
if STOP or state['consecutive_nonfinite'] >= config['max_consecutive_nonfinite']:
|
| 339 |
+
stopping_cause = 'user_interrupt' if STOP else 'nonfinite_limit'
|
| 340 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter()-start_time
|
| 341 |
+
save('latest.pt')
|
| 342 |
+
emit({'event':'stopped','stopping_cause':stopping_cause,'step':state['step']})
|
| 343 |
+
log.close()
|
| 344 |
+
return False
|
| 345 |
+
state['epoch'] += 1
|
| 346 |
+
state['offset'] = 0
|
| 347 |
+
order = None
|
| 348 |
+
if state['last_eval_step'] != state['step']:
|
| 349 |
+
evaluate_and_save()
|
| 350 |
+
else:
|
| 351 |
+
save('latest.pt')
|
| 352 |
+
# Final held-out scores use the best validation-selected checkpoint.
|
| 353 |
+
best = torch.load(run_dir/'best.pt',map_location='cpu',weights_only=False,mmap=True)
|
| 354 |
+
model.load_state_dict(best['model'],strict=True)
|
| 355 |
+
best_step = best['state']['step']
|
| 356 |
+
del best
|
| 357 |
+
after = evaluate(model,data[heldout],evaluation_batch)
|
| 358 |
+
(run_dir/'after.json').write_text(json.dumps(after,indent=2),encoding='utf-8')
|
| 359 |
+
rows = raw_rows(task,heldout)
|
| 360 |
+
examples = []
|
| 361 |
+
for row_index,(row,pb,pa) in enumerate(zip(rows,before['predictions'],after['predictions'])):
|
| 362 |
+
example = {'id':f'{task}/{heldout}/{row_index}','input':row.get('text',row.get('article','')),
|
| 363 |
+
'prompt':row.get('question',''),'options':row.get('options',['negative','positive']),
|
| 364 |
+
'gold':row['label'],'predicted_before':pb,'predicted_after':pa,
|
| 365 |
+
'correct_before':pb==row['label'],'correct_after':pa==row['label']}
|
| 366 |
+
examples.append(example)
|
| 367 |
+
(run_dir/'heldout_predictions.json').write_text(json.dumps(examples,ensure_ascii=False,indent=2),encoding='utf-8')
|
| 368 |
+
probe_count = 100 if task=='sst2' else 40
|
| 369 |
+
probe = {'before_correct':sum(e['correct_before'] for e in examples[:probe_count]),
|
| 370 |
+
'after_correct':sum(e['correct_after'] for e in examples[:probe_count]),'total':probe_count}
|
| 371 |
+
state['elapsed_seconds'] = elapsed_prior+time.perf_counter()-start_time
|
| 372 |
+
summary = {'id':task,'name':'SST-2 sentiment' if task=='sst2' else 'RACE middle-school reading',
|
| 373 |
+
'train_examples':n_train,'validation_examples':len(data['validation']['labels']),
|
| 374 |
+
'test_examples':len(data['test']['labels']) if task!='sst2' else 1821,
|
| 375 |
+
'heldout_split':heldout,'baseline':{'label':'Before task training (same classifier)',**full_summary(before)},
|
| 376 |
+
'finetuned':full_summary(after),'chance_accuracy':0.5 if task=='sst2' else 0.25,
|
| 377 |
+
'majority_accuracy':after['majority_accuracy'],'paired_earlier_probe':probe,
|
| 378 |
+
'training':{'epochs':config['epochs'],'max_steps':total_steps,'steps':state['step'],
|
| 379 |
+
'batch_size':config['effective_batch_size'],'microbatch_size':task_config['microbatch_size'],
|
| 380 |
+
'gradient_accumulation':config['effective_batch_size']//task_config['microbatch_size'],
|
| 381 |
+
'learning_rate':config['learning_rate'],'max_length':config['max_length'],
|
| 382 |
+
'lm_coefficient':config['lm_coefficient'],'stopping_cause':stopping_cause,
|
| 383 |
+
'classifier_dropout':config['classifier_dropout'],'native_backbone_dropout':base.config.dropout,
|
| 384 |
+
'nonfinite_updates':state['nonfinite_updates'],'best_step':best_step,
|
| 385 |
+
'peak_allocated_gib':round(torch.cuda.max_memory_allocated()/1024**3,3),
|
| 386 |
+
'trainable_parameters':sum(p.numel() for p in model.parameters()),
|
| 387 |
+
'head':'768-to-2 linear' if task=='sst2' else 'shared 768-to-1 linear over four answers',
|
| 388 |
+
'checkpoint_selection':'highest validation accuracy, lower validation classification loss breaks ties',
|
| 389 |
+
'details':'Full native backbone trained. Zero decay on oscillator and 1-D parameters. Linear GPT-1 warmup/decay, AdamW, gradient clip1.0. Original backbone dropout retained; classifier dropout0.1. Three unused padding-vocabulary slots initialized for task markers.'},
|
| 390 |
+
'examples':examples,'checkpoint':{'name':str((run_dir/'best.pt').relative_to(ROOT)),
|
| 391 |
+
'sha256':digest(run_dir/'best.pt')},
|
| 392 |
+
'elapsed_seconds':state['elapsed_seconds']}
|
| 393 |
+
(run_dir/'summary.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2),encoding='utf-8')
|
| 394 |
+
emit({'event':'complete','step':state['step'],'best_step':best_step,'stopping_cause':stopping_cause,
|
| 395 |
+
'heldout':heldout,'before':full_summary(before),'after':full_summary(after),
|
| 396 |
+
'paired_probe':probe,'elapsed_seconds':round(state['elapsed_seconds'],1)})
|
| 397 |
+
log.close()
|
| 398 |
+
del model,base,optimizer,data
|
| 399 |
+
gc.collect();torch.cuda.empty_cache()
|
| 400 |
+
return True
|
| 401 |
+
|
| 402 |
+
def write_comparison():
|
| 403 |
+
tasks=[]
|
| 404 |
+
for task in ['sst2','race_middle']:
|
| 405 |
+
paths=sorted((ROOT/'runs').glob(f'*_{task}/summary.json'),key=lambda path:path.stat().st_mtime)
|
| 406 |
+
if not paths:
|
| 407 |
+
return
|
| 408 |
+
tasks.append(json.loads(paths[-1].read_text(encoding='utf-8')))
|
| 409 |
+
data={'date':datetime.now(ZoneInfo('Europe/London')).isoformat(),
|
| 410 |
+
'scope':'Full SST-2 training and validation; full RACE-middle training, validation selection and held-out test. Historical GPT-1 figures use different training coverage and test sets.',
|
| 411 |
+
'method':'Two independent full-model adaptations from original pretrained WiggleGPT. Classifier cross-entropy plus0.5 times token-average auxiliary language-model cross-entropy, with GPT-1-style input markers and linear task heads.',
|
| 412 |
+
'control_method':'Identical task inputs and initialized classifier evaluated before task training; after scores use validation-selected trained checkpoint. Earlier language-model likelihood probes are separately identified.',
|
| 413 |
+
'historical':{'gpt1_race_accuracy':59.0,'gpt1_race_middle_accuracy':62.9,'gpt1_sst2_accuracy':91.3,
|
| 414 |
+
'source_url':'https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf',
|
| 415 |
+
'note':'GPT-1 SST-2 91.3% is GLUE test, while this run reports public validation. GPT-1 RACE-M62.9% used combined RACE-middle/high training, while Wiggle used middle-only. Historical rows give context and are not a matched-model replication.'},
|
| 416 |
+
'tasks':tasks,'elapsed_seconds':sum(t['elapsed_seconds'] for t in tasks)}
|
| 417 |
+
(ROOT/'comparison.json').write_text(json.dumps(data,ensure_ascii=False,indent=2),encoding='utf-8')
|
| 418 |
+
|
| 419 |
+
def main():
|
| 420 |
+
parser=argparse.ArgumentParser(description=__doc__)
|
| 421 |
+
parser.add_argument('--task',choices=['all','sst2','race_middle'],default='all')
|
| 422 |
+
parser.add_argument('--resume',type=Path)
|
| 423 |
+
args=parser.parse_args()
|
| 424 |
+
config=json.loads((ROOT/'train_config.json').read_text(encoding='utf-8'))
|
| 425 |
+
torch.set_num_threads(config['cpu_threads'])
|
| 426 |
+
try:
|
| 427 |
+
import psutil
|
| 428 |
+
psutil.Process().nice(psutil.BELOW_NORMAL_PRIORITY_CLASS)
|
| 429 |
+
except ImportError:
|
| 430 |
+
print('psutil unavailable; continuing with two CPU threads.',flush=True)
|
| 431 |
+
if not torch.cuda.is_available():
|
| 432 |
+
raise RuntimeError('CUDA required; CPU model execution is disabled.')
|
| 433 |
+
torch.cuda.set_device(0)
|
| 434 |
+
torch.set_float32_matmul_precision('high')
|
| 435 |
+
signal.signal(signal.SIGINT,stop_requested)
|
| 436 |
+
if args.resume:
|
| 437 |
+
resume_meta=torch.load(args.resume,map_location='cpu',weights_only=False,mmap=True)
|
| 438 |
+
selected=[resume_meta['task']]
|
| 439 |
+
del resume_meta
|
| 440 |
+
else:
|
| 441 |
+
selected=['sst2','race_middle'] if args.task=='all' else [args.task]
|
| 442 |
+
for task in selected:
|
| 443 |
+
if not run_task(task,config,args.resume):
|
| 444 |
+
return
|
| 445 |
+
write_comparison()
|
| 446 |
+
print('All requested task adaptations complete.',flush=True)
|
| 447 |
+
|
| 448 |
+
if __name__=='__main__':
|
| 449 |
+
main()
|
task_finetuning/results/train_tasks_v2_original_source.txt
ADDED
|
@@ -0,0 +1,539 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Full-model GPT-1-style task fine-tuning of original WiggleGPT, version 2.
|
| 2 |
+
|
| 3 |
+
Native model_bio.py is imported unchanged. A final-state classifier is added;
|
| 4 |
+
its language-model objective uses the native tied projection. No CPU forwards.
|
| 5 |
+
CPU length plans remove trailing padding; auxiliary LM projection skips padding.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import argparse
|
| 10 |
+
from collections import Counter
|
| 11 |
+
from datetime import datetime
|
| 12 |
+
from zoneinfo import ZoneInfo
|
| 13 |
+
import gc
|
| 14 |
+
import hashlib
|
| 15 |
+
import json
|
| 16 |
+
import math
|
| 17 |
+
import os
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
import random
|
| 20 |
+
import shutil
|
| 21 |
+
import signal
|
| 22 |
+
import sys
|
| 23 |
+
import time
|
| 24 |
+
|
| 25 |
+
ROOT = Path(__file__).resolve().parent
|
| 26 |
+
WORKSPACE = ROOT.parents[2]
|
| 27 |
+
NATIVE = WORKSPACE / 'Main folders' / 'WiggleGPT v3' / 'WiggleGPT'
|
| 28 |
+
sys.path.insert(0, str(WORKSPACE / 'WiggleGPT-Gosport'))
|
| 29 |
+
sys.path.insert(0, str(NATIVE))
|
| 30 |
+
from wiggle_assistant import load_model
|
| 31 |
+
import model_bio
|
| 32 |
+
import torch
|
| 33 |
+
from torch import nn
|
| 34 |
+
from torch.nn import functional as F
|
| 35 |
+
|
| 36 |
+
STOP = False
|
| 37 |
+
MARKERS = {'start': 50257, 'delimiter': 50258, 'classify': 50259, 'pad': 50256}
|
| 38 |
+
OPTIMIZATIONS = {
|
| 39 |
+
'version': 2,
|
| 40 |
+
'training_padding': 'CPU epoch plan crops each unchanged microbatch to its longest candidate, rounded up to 16 tokens (minimum 16).',
|
| 41 |
+
'auxiliary_lm': 'Project only valid next-token positions; weight each token by the inverse length of its own candidate, then average all candidates.',
|
| 42 |
+
'device_synchronization': 'One order copy to CPU per epoch; CPU-known output_size avoids synchronization when building packed CUDA indices.',
|
| 43 |
+
'question_order_and_effective_batch': 'Original seeded CUDA randperm, effective batch and microbatch membership retained.',
|
| 44 |
+
'native_backbone': 'Unchanged native model_bio.py and configured gradient checkpointing retained.',
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
def stop_requested(signum, frame):
|
| 48 |
+
global STOP
|
| 49 |
+
STOP = True
|
| 50 |
+
print('Stop requested; finishing the update and saving latest.pt.', flush=True)
|
| 51 |
+
|
| 52 |
+
def digest(path):
|
| 53 |
+
with Path(path).open('rb') as stream:
|
| 54 |
+
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
| 55 |
+
|
| 56 |
+
class TaskModel(nn.Module):
|
| 57 |
+
def __init__(self, base, task, dropout):
|
| 58 |
+
super().__init__()
|
| 59 |
+
self.base = base
|
| 60 |
+
self.task = task
|
| 61 |
+
self.dropout = dropout
|
| 62 |
+
# Added linear classifier is the requested GPT-1 task head, not a backbone replacement.
|
| 63 |
+
with torch.device('cuda:0'):
|
| 64 |
+
self.classifier = nn.Linear(base.config.n_embd, 2 if task == 'sst2' else 1)
|
| 65 |
+
nn.init.normal_(self.classifier.weight, mean=0.0, std=0.02)
|
| 66 |
+
nn.init.zeros_(self.classifier.bias)
|
| 67 |
+
self.hidden = None
|
| 68 |
+
self.last_loss_verification = None
|
| 69 |
+
self.base.transformer.ln_f.register_forward_hook(self.capture_hidden)
|
| 70 |
+
|
| 71 |
+
def capture_hidden(self, module, args, output):
|
| 72 |
+
self.hidden = output
|
| 73 |
+
|
| 74 |
+
def forward(self, tokens, lengths, auxiliary=False, packed_lm_tokens=None,
|
| 75 |
+
verify_packed=False):
|
| 76 |
+
batch, choices, width = tokens.shape
|
| 77 |
+
flat = tokens.reshape(batch * choices, width)
|
| 78 |
+
self.base(flat)
|
| 79 |
+
hidden = self.hidden
|
| 80 |
+
self.hidden = None
|
| 81 |
+
positions = lengths.reshape(-1) - 1
|
| 82 |
+
features = hidden[torch.arange(batch * choices, device=flat.device), positions]
|
| 83 |
+
features = features.reshape(batch, choices, -1)
|
| 84 |
+
# GPT-1's multiple-choice head shares a dropout feature mask across options.
|
| 85 |
+
if self.training and self.dropout:
|
| 86 |
+
mask = F.dropout(torch.ones((batch, 1, features.shape[-1]), device=features.device,
|
| 87 |
+
dtype=features.dtype), p=self.dropout, training=True)
|
| 88 |
+
features = features * mask
|
| 89 |
+
scores = self.classifier(features).squeeze(1) if self.task == 'sst2' else self.classifier(features).squeeze(-1)
|
| 90 |
+
lm_loss = None
|
| 91 |
+
if auxiliary:
|
| 92 |
+
# Average within each candidate sequence, then across all candidates.
|
| 93 |
+
# output_size is supplied by the CPU length plan, avoiding a CUDA
|
| 94 |
+
# synchronization to discover the size of a boolean-index result.
|
| 95 |
+
if packed_lm_tokens is None:
|
| 96 |
+
raise ValueError('Auxiliary LM requires the CPU-planned valid-token count.')
|
| 97 |
+
counts = lengths.reshape(-1) - 1
|
| 98 |
+
sequence_ids = torch.repeat_interleave(
|
| 99 |
+
torch.arange(batch * choices, device=flat.device), counts,
|
| 100 |
+
output_size=packed_lm_tokens)
|
| 101 |
+
starts = counts.cumsum(0) - counts
|
| 102 |
+
token_positions = (torch.arange(packed_lm_tokens, device=flat.device)
|
| 103 |
+
- starts[sequence_ids])
|
| 104 |
+
flat_positions = sequence_ids * width + token_positions
|
| 105 |
+
lm_inputs = hidden.reshape(-1, hidden.shape[-1]).index_select(0, flat_positions)
|
| 106 |
+
targets = flat.reshape(-1).index_select(0, flat_positions + 1).long()
|
| 107 |
+
logits = self.base.lm_head(lm_inputs)
|
| 108 |
+
per_token = F.cross_entropy(logits.float(), targets, reduction='none')
|
| 109 |
+
lm_loss = (per_token / counts[sequence_ids]).sum() / (batch * choices)
|
| 110 |
+
if verify_packed:
|
| 111 |
+
# Both paths use these same actual CUDA hidden states, tied
|
| 112 |
+
# projection, autocast dtype and per-candidate normalization.
|
| 113 |
+
# Limit the comparison to one real candidate to keep the
|
| 114 |
+
# optional once-only verification below the desktop VRAM cap.
|
| 115 |
+
with torch.no_grad():
|
| 116 |
+
dense_logits = self.base.lm_head(hidden[:1, :-1].detach())
|
| 117 |
+
dense_targets = flat[:1, 1:].long()
|
| 118 |
+
dense_tokens = F.cross_entropy(
|
| 119 |
+
dense_logits.float().reshape(-1, dense_logits.shape[-1]),
|
| 120 |
+
dense_targets.reshape(-1), reduction='none').view_as(dense_targets)
|
| 121 |
+
dense_mask = torch.arange(width - 1, device=flat.device)[None, :] < counts[:1, None]
|
| 122 |
+
dense_loss = ((dense_tokens * dense_mask).sum(-1)
|
| 123 |
+
/ counts[:1].clamp_min(1)).mean()
|
| 124 |
+
packed_first = ((per_token.detach() / counts[sequence_ids])
|
| 125 |
+
* (sequence_ids == 0)).sum()
|
| 126 |
+
values = torch.stack((dense_loss, packed_first,
|
| 127 |
+
(dense_loss - packed_first).abs())).cpu().tolist() # lint: ok COMPLEX-ABS real scalar CE losses
|
| 128 |
+
self.last_loss_verification = {
|
| 129 |
+
'passed': math.isclose(values[0], values[1], rel_tol=1e-4, abs_tol=1e-5),
|
| 130 |
+
'dense_loss': values[0], 'packed_loss': values[1],
|
| 131 |
+
'absolute_difference': values[2], 'relative_tolerance': 1e-4,
|
| 132 |
+
'absolute_tolerance': 1e-5, 'candidates_checked': 1,
|
| 133 |
+
'actual_training_candidates': batch * choices,
|
| 134 |
+
'packed_lm_tokens': packed_lm_tokens, 'cropped_width': width,
|
| 135 |
+
'projection_dtype': str(logits.dtype),
|
| 136 |
+
'device': str(hidden.device),
|
| 137 |
+
'comparison': 'Same actual training hidden states; dense masked CE versus packed valid-position CE, before the first optimizer update.',
|
| 138 |
+
}
|
| 139 |
+
return scores.float(), lm_loss
|
| 140 |
+
|
| 141 |
+
def load_data(task, device):
|
| 142 |
+
prefix = 'sst2' if task == 'sst2' else 'race_middle'
|
| 143 |
+
splits = ['train', 'validation'] + (['test'] if task != 'sst2' else [])
|
| 144 |
+
data = {}
|
| 145 |
+
for split in splits:
|
| 146 |
+
path = ROOT / 'data' / f'{prefix}_{split}_encoded.pt'
|
| 147 |
+
if not path.exists():
|
| 148 |
+
# Preparation script may use the compact .pt suffix.
|
| 149 |
+
path = ROOT / 'data' / f'{prefix}_{split}.pt'
|
| 150 |
+
cache = torch.load(path, map_location='cpu', weights_only=False, mmap=True)
|
| 151 |
+
# Both conventional names are accepted; contents stay unchanged.
|
| 152 |
+
ids = cache.get('input_ids', cache.get('tokens'))
|
| 153 |
+
data[split] = {'input_ids': ids.to(device), 'lengths': cache['lengths'].to(device),
|
| 154 |
+
'lengths_cpu': cache['lengths'].clone(),
|
| 155 |
+
'labels': cache['labels'].to(device), 'cache_path': str(path),
|
| 156 |
+
'metadata': {k:v for k,v in cache.items() if not torch.is_tensor(v)}}
|
| 157 |
+
return data
|
| 158 |
+
|
| 159 |
+
def select_batch(data, index, width=None):
|
| 160 |
+
tokens = data['input_ids'][index]
|
| 161 |
+
if width is not None:
|
| 162 |
+
tokens = tokens[..., :width]
|
| 163 |
+
return tokens, data['lengths'][index], data['labels'][index]
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def plan_epoch_microbatches(data, order, effective_batch_size, microbatch_size):
|
| 167 |
+
"""Plan CPU scalar lengths once, preserving the saved CUDA question order."""
|
| 168 |
+
ordered_lengths = data['lengths_cpu'].index_select(0, order.detach().cpu())
|
| 169 |
+
cached_width = data['input_ids'].shape[-1]
|
| 170 |
+
plan = {}
|
| 171 |
+
for start in range(0, len(order), effective_batch_size):
|
| 172 |
+
end = min(len(order), start + effective_batch_size)
|
| 173 |
+
for position in range(start, end, microbatch_size):
|
| 174 |
+
lengths = ordered_lengths[position:min(end, position + microbatch_size)]
|
| 175 |
+
width = min(cached_width, max(16, math.ceil(int(lengths.max()) / 16) * 16))
|
| 176 |
+
plan[position] = (width, int((lengths - 1).sum()))
|
| 177 |
+
return plan
|
| 178 |
+
|
| 179 |
+
def evaluate(model, data, batch_size, fixed_indices=None):
|
| 180 |
+
model.eval()
|
| 181 |
+
predictions, losses = [], []
|
| 182 |
+
if fixed_indices is None:
|
| 183 |
+
fixed_indices = torch.arange(data['labels'].shape[0], device='cuda:0')
|
| 184 |
+
# AMP-COMPLEX: native weights/activations are real; native RoPE explicitly casts its complex rotation to float32.
|
| 185 |
+
with torch.inference_mode(), torch.autocast('cuda', dtype=torch.bfloat16): # lint: ok AMP-COMPLEX real native weights; RoPE explicitly uses float32 complex rotation
|
| 186 |
+
for start in range(0, len(fixed_indices), batch_size):
|
| 187 |
+
indices = fixed_indices[start:start + batch_size]
|
| 188 |
+
x, lengths, labels = select_batch(data, indices)
|
| 189 |
+
logits, _ = model(x, lengths)
|
| 190 |
+
predictions.append(logits.argmax(-1))
|
| 191 |
+
losses.append(F.cross_entropy(logits, labels, reduction='sum'))
|
| 192 |
+
predicted = torch.cat(predictions)
|
| 193 |
+
labels = data['labels'][fixed_indices]
|
| 194 |
+
correct = int((predicted == labels).sum())
|
| 195 |
+
counts = torch.bincount(labels, minlength=2 if model.task == 'sst2' else 4).cpu().tolist()
|
| 196 |
+
return {'correct': correct, 'total': len(labels), 'accuracy': correct / len(labels),
|
| 197 |
+
'loss': float(torch.stack(losses).sum() / len(labels)),
|
| 198 |
+
'predictions': predicted.cpu().tolist(), 'gold_label_counts': counts,
|
| 199 |
+
'majority_accuracy': max(counts) / len(labels)}
|
| 200 |
+
|
| 201 |
+
def optimizer_for(model, config):
|
| 202 |
+
decay, no_decay = [], []
|
| 203 |
+
protected = ('omega', 'phi', 'baseline', 'norm', 'bias', 'scale', 'phase')
|
| 204 |
+
for name, parameter in model.named_parameters():
|
| 205 |
+
if parameter.ndim >= 2 and not parameter.is_complex() and not any(word in name for word in protected):
|
| 206 |
+
decay.append(parameter)
|
| 207 |
+
else:
|
| 208 |
+
no_decay.append(parameter)
|
| 209 |
+
print(f'Decay: {sum(p.numel() for p in decay):,}; protected/no decay: {sum(p.numel() for p in no_decay):,}', flush=True)
|
| 210 |
+
return torch.optim.AdamW([{'params':decay, 'weight_decay':config['weight_decay']},
|
| 211 |
+
{'params':no_decay, 'weight_decay':0.0}],
|
| 212 |
+
lr=config['learning_rate'], betas=tuple(config['betas']), eps=config['epsilon'], foreach=True)
|
| 213 |
+
|
| 214 |
+
def schedule(step, total, config):
|
| 215 |
+
warmup = max(1, round(total * config['warmup_fraction']))
|
| 216 |
+
if step < warmup:
|
| 217 |
+
return config['learning_rate'] * (step + 1) / warmup
|
| 218 |
+
return config['learning_rate'] * max(0.0, (total - step) / max(1, total - warmup))
|
| 219 |
+
|
| 220 |
+
def atomic_save(path, payload):
|
| 221 |
+
temporary = path.with_suffix('.pending.pt')
|
| 222 |
+
torch.save(payload, temporary)
|
| 223 |
+
temporary.replace(path)
|
| 224 |
+
|
| 225 |
+
def rng_state(generator):
|
| 226 |
+
return {'python': random.getstate(), 'torch': torch.get_rng_state(),
|
| 227 |
+
'cuda': torch.cuda.get_rng_state(0), 'batch_generator': generator.get_state()}
|
| 228 |
+
|
| 229 |
+
def restore_rng(state, generator):
|
| 230 |
+
random.setstate(state['python'])
|
| 231 |
+
torch.set_rng_state(state['torch'].cpu())
|
| 232 |
+
torch.cuda.set_rng_state(state['cuda'].cpu(), 0)
|
| 233 |
+
generator.set_state(state['batch_generator'].cpu())
|
| 234 |
+
|
| 235 |
+
def full_summary(result):
|
| 236 |
+
return {k:v for k,v in result.items() if k != 'predictions'}
|
| 237 |
+
|
| 238 |
+
def raw_rows(task, split):
|
| 239 |
+
path = ROOT / 'data' / f'{task}_{split}.jsonl'
|
| 240 |
+
return [json.loads(line) for line in path.read_text(encoding='utf-8').splitlines()]
|
| 241 |
+
|
| 242 |
+
def run_task(task, config, resume=None, verify_packed_loss_once=False):
|
| 243 |
+
task_config = config['tasks'][task]
|
| 244 |
+
random.seed(config['seed'])
|
| 245 |
+
torch.manual_seed(config['seed'])
|
| 246 |
+
torch.cuda.manual_seed_all(config['seed'])
|
| 247 |
+
base, native_config, encoder, metadata = load_model((ROOT / config['starting_checkpoint']).resolve())
|
| 248 |
+
free, total = torch.cuda.mem_get_info()
|
| 249 |
+
cap = min(int(config['max_process_gib'] * 1024**3),
|
| 250 |
+
total - int(config['desktop_reserve_gib'] * 1024**3),
|
| 251 |
+
free + torch.cuda.memory_reserved() - int(0.5 * 1024**3))
|
| 252 |
+
torch.cuda.set_per_process_memory_fraction(cap / total, 0)
|
| 253 |
+
base.config.gradient_checkpointing = task_config['gradient_checkpointing']
|
| 254 |
+
metadata['checkpoint_sha256'] = digest((ROOT / config['starting_checkpoint']).resolve())
|
| 255 |
+
model = TaskModel(base, task, config['classifier_dropout'])
|
| 256 |
+
# Reuse three existing padding slots; real GPT-2 IDs and tokenizer are unchanged.
|
| 257 |
+
with torch.no_grad():
|
| 258 |
+
marker_ids = torch.tensor([MARKERS['start'],MARKERS['delimiter'],MARKERS['classify']], device='cuda:0')
|
| 259 |
+
base.transformer.wte.weight.index_copy_(0, marker_ids,
|
| 260 |
+
torch.randn((3, base.config.n_embd), device='cuda:0') * 0.02)
|
| 261 |
+
optimizer = optimizer_for(model, config)
|
| 262 |
+
data = load_data(task, 'cuda:0')
|
| 263 |
+
n_train = data['train']['labels'].shape[0]
|
| 264 |
+
per_epoch = math.ceil(n_train / config['effective_batch_size'])
|
| 265 |
+
total_steps = per_epoch * config['epochs']
|
| 266 |
+
generator = torch.Generator(device='cuda:0').manual_seed(config['seed'] + 123)
|
| 267 |
+
if resume:
|
| 268 |
+
checkpoint = torch.load(resume, map_location='cpu', weights_only=False, mmap=True)
|
| 269 |
+
model.load_state_dict(checkpoint['model'], strict=True)
|
| 270 |
+
optimizer.load_state_dict(checkpoint['optimizer'])
|
| 271 |
+
state = checkpoint['state']
|
| 272 |
+
order = checkpoint['order'].to('cuda:0') if checkpoint['order'] is not None else None
|
| 273 |
+
restore_rng(checkpoint['rng'], generator)
|
| 274 |
+
run_dir = Path(resume).resolve().parent
|
| 275 |
+
del checkpoint
|
| 276 |
+
else:
|
| 277 |
+
run_dir = ROOT / 'runs' / f'{Path(__file__).stem}_{datetime.now(ZoneInfo("Europe/London")).strftime("%Y-%m-%d_%H-%M-%S")}_{task}'
|
| 278 |
+
run_dir.mkdir(parents=True, exist_ok=False)
|
| 279 |
+
state = {'step':0, 'epoch':0, 'offset':0, 'best_accuracy':-1.0, 'best_loss':float('inf'),
|
| 280 |
+
'best_step':0, 'nonfinite_updates':0, 'consecutive_nonfinite':0, 'last_eval_step':-1,
|
| 281 |
+
'elapsed_seconds':0.0, 'lr_scale':1.0, 'packed_loss_verification':None}
|
| 282 |
+
order = None
|
| 283 |
+
(run_dir / 'config.json').write_text(json.dumps(config, indent=2), encoding='utf-8')
|
| 284 |
+
shutil.copy2(__file__, run_dir / Path(__file__).name)
|
| 285 |
+
shutil.copy2(model_bio.__file__, run_dir / 'model_bio.py')
|
| 286 |
+
provenance = {'model_source_sha256':digest(model_bio.__file__), 'trainer_sha256':digest(__file__),
|
| 287 |
+
'starting_checkpoint':metadata, 'torch':torch.__version__, 'markers':MARKERS,
|
| 288 |
+
'native_backbone_dropout':base.config.dropout,
|
| 289 |
+
'data':{split:d['metadata'] for split,d in data.items()},
|
| 290 |
+
'all_backbone_parameters_train':True, 'optimizations':OPTIMIZATIONS}
|
| 291 |
+
(run_dir/'provenance.json').write_text(json.dumps(provenance, indent=2), encoding='utf-8')
|
| 292 |
+
log = (run_dir / (run_dir.name + '.log')).open('a', encoding='utf-8', buffering=1)
|
| 293 |
+
def emit(event):
|
| 294 |
+
event.update(task=task)
|
| 295 |
+
line = json.dumps(event, ensure_ascii=False)
|
| 296 |
+
print(line, flush=True)
|
| 297 |
+
log.write(line + '\n')
|
| 298 |
+
with (run_dir/'metrics.jsonl').open('a',encoding='utf-8') as stream:
|
| 299 |
+
stream.write(line + '\n')
|
| 300 |
+
def save(name):
|
| 301 |
+
atomic_save(run_dir/name, {'model':model.state_dict(), 'optimizer':optimizer.state_dict(),
|
| 302 |
+
'state':dict(state), 'order':order, 'rng':rng_state(generator), 'config':config,
|
| 303 |
+
'task':task, 'model_args':vars(base.config), 'markers':MARKERS,
|
| 304 |
+
'source_hashes':{'model':digest(model_bio.__file__),'trainer':digest(__file__)}})
|
| 305 |
+
evaluation_batch = 16 if task == 'sst2' else 4
|
| 306 |
+
heldout = 'validation' if task == 'sst2' else 'test'
|
| 307 |
+
before_path = run_dir/'before.json'
|
| 308 |
+
start_time = time.perf_counter()
|
| 309 |
+
elapsed_prior = state['elapsed_seconds']
|
| 310 |
+
emit({'event':'start','step':state['step'],'train_examples':n_train,'epochs':config['epochs'],
|
| 311 |
+
'total_steps':total_steps,'microbatch':task_config['microbatch_size'],
|
| 312 |
+
'effective_batch':config['effective_batch_size'],'memory_cap_gib':round(cap/1024**3,2),
|
| 313 |
+
'trainable_parameters':sum(p.numel() for p in model.parameters() if p.requires_grad),
|
| 314 |
+
'device':torch.cuda.get_device_name(0), 'optimizations':OPTIMIZATIONS,
|
| 315 |
+
'verify_packed_loss_once':verify_packed_loss_once})
|
| 316 |
+
if not before_path.exists():
|
| 317 |
+
before = evaluate(model, data[heldout], evaluation_batch)
|
| 318 |
+
before_path.write_text(json.dumps(before,indent=2),encoding='utf-8')
|
| 319 |
+
emit({'event':'before','heldout':heldout,**full_summary(before)})
|
| 320 |
+
else:
|
| 321 |
+
before = json.loads(before_path.read_text(encoding='utf-8'))
|
| 322 |
+
train_fixed = torch.arange(min(128,n_train), device='cuda:0')
|
| 323 |
+
def evaluate_and_save():
|
| 324 |
+
validation = evaluate(model, data['validation'], evaluation_batch)
|
| 325 |
+
train_eval = evaluate(model, data['train'], evaluation_batch, train_fixed)
|
| 326 |
+
changed = (validation['accuracy'] > state['best_accuracy'] or
|
| 327 |
+
(validation['accuracy'] == state['best_accuracy'] and validation['loss'] < state['best_loss']))
|
| 328 |
+
if changed:
|
| 329 |
+
state['best_accuracy'] = validation['accuracy']
|
| 330 |
+
state['best_loss'] = validation['loss']
|
| 331 |
+
state['best_step'] = state['step']
|
| 332 |
+
state['last_eval_step'] = state['step']
|
| 333 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter() - start_time
|
| 334 |
+
# Once-per-evaluation diagnostic over twelve oscillator tensors, not Python-built training batches.
|
| 335 |
+
omegas = torch.cat([p.detach().reshape(-1) for name,p in model.named_parameters() if name.endswith('.omega')]) # lint: ok STACK-COMPREHENSION once-per-evaluation statistics over twelve oscillator tensors
|
| 336 |
+
emit({'event':'eval','step':state['step'],'epoch':state['epoch'],'offset':state['offset'],
|
| 337 |
+
'train':full_summary(train_eval),'validation':full_summary(validation),
|
| 338 |
+
'best_step':state['best_step'],'omega_std':float(omegas.std(correction=0)),
|
| 339 |
+
'peak_allocated_gib':round(torch.cuda.max_memory_allocated()/1024**3,3),
|
| 340 |
+
'sample':{'gold':int(data['validation']['labels'][0]),'predicted':validation['predictions'][0]},
|
| 341 |
+
'elapsed_seconds':round(state['elapsed_seconds'],1)})
|
| 342 |
+
if changed:
|
| 343 |
+
save('best.pt')
|
| 344 |
+
save('latest.pt')
|
| 345 |
+
model.train()
|
| 346 |
+
if state['last_eval_step'] < 0:
|
| 347 |
+
evaluate_and_save()
|
| 348 |
+
model.train()
|
| 349 |
+
accumulated = torch.zeros(3,device='cuda:0')
|
| 350 |
+
logged_steps = 0
|
| 351 |
+
log_start = time.perf_counter()
|
| 352 |
+
stopping_cause = 'epochs_complete'
|
| 353 |
+
while state['epoch'] < config['epochs']:
|
| 354 |
+
if order is None:
|
| 355 |
+
order = torch.randperm(n_train,device='cuda:0',generator=generator)
|
| 356 |
+
# Reconstructed identically from saved order/CPU lengths after resume;
|
| 357 |
+
# no extra RNG draws and no per-microbatch CUDA-to-CPU length transfers.
|
| 358 |
+
microbatch_plan = plan_epoch_microbatches(
|
| 359 |
+
data['train'], order, config['effective_batch_size'], task_config['microbatch_size'])
|
| 360 |
+
while state['offset'] < n_train:
|
| 361 |
+
end = min(n_train,state['offset']+config['effective_batch_size'])
|
| 362 |
+
indices = order[state['offset']:end]
|
| 363 |
+
effective = len(indices)
|
| 364 |
+
lr = schedule(state['step'], total_steps, config) * state['lr_scale']
|
| 365 |
+
for group in optimizer.param_groups:
|
| 366 |
+
group['lr'] = lr
|
| 367 |
+
optimizer.zero_grad(set_to_none=True)
|
| 368 |
+
batch_values = torch.zeros(3,device='cuda:0')
|
| 369 |
+
for position in range(0,effective,task_config['microbatch_size']):
|
| 370 |
+
micro = indices[position:position+task_config['microbatch_size']]
|
| 371 |
+
width, packed_lm_tokens = microbatch_plan[state['offset'] + position]
|
| 372 |
+
x,lengths,labels = select_batch(data['train'],micro,width)
|
| 373 |
+
verify_this = (verify_packed_loss_once and task != 'sst2'
|
| 374 |
+
and not state.get('packed_loss_verification'))
|
| 375 |
+
# AMP-COMPLEX: real-valued model; native rotary complex operations remain explicitly float32.
|
| 376 |
+
with torch.autocast('cuda',dtype=torch.bfloat16): # lint: ok AMP-COMPLEX real native weights; RoPE explicitly uses float32 complex rotation
|
| 377 |
+
logits,lm_loss = model(x,lengths,auxiliary=True,
|
| 378 |
+
packed_lm_tokens=packed_lm_tokens,verify_packed=verify_this)
|
| 379 |
+
classification_loss = F.cross_entropy(logits,labels)
|
| 380 |
+
loss = classification_loss + config['lm_coefficient'] * lm_loss
|
| 381 |
+
weighted = loss * (len(micro)/effective)
|
| 382 |
+
if verify_this:
|
| 383 |
+
verification = model.last_loss_verification
|
| 384 |
+
state['packed_loss_verification'] = verification
|
| 385 |
+
(run_dir/'packed_loss_verification.json').write_text(
|
| 386 |
+
json.dumps(verification,indent=2),encoding='utf-8')
|
| 387 |
+
emit({'event':'packed_loss_verification','step':state['step'],**verification})
|
| 388 |
+
if not verification['passed']:
|
| 389 |
+
raise RuntimeError('Packed auxiliary LM loss did not match dense masked loss; no optimizer update made.')
|
| 390 |
+
weighted.backward()
|
| 391 |
+
batch_values += torch.stack((loss.detach(),classification_loss.detach(),lm_loss.detach())) * (len(micro)/effective)
|
| 392 |
+
norm = torch.nn.utils.clip_grad_norm_(model.parameters(),config['grad_clip'])
|
| 393 |
+
if torch.isfinite(norm):
|
| 394 |
+
optimizer.step()
|
| 395 |
+
state['consecutive_nonfinite'] = 0
|
| 396 |
+
else:
|
| 397 |
+
state['nonfinite_updates'] += 1
|
| 398 |
+
state['consecutive_nonfinite'] += 1
|
| 399 |
+
if state['consecutive_nonfinite'] >= 2:
|
| 400 |
+
state['lr_scale'] *= 0.5
|
| 401 |
+
emit({'event':'nonfinite_update_skipped','step':state['step']+1,
|
| 402 |
+
'count':state['nonfinite_updates'],'lr_scale':state['lr_scale']})
|
| 403 |
+
state['step'] += 1
|
| 404 |
+
state['offset'] = end
|
| 405 |
+
accumulated += batch_values
|
| 406 |
+
logged_steps += 1
|
| 407 |
+
if state['step'] == 1 or state['step'] % config['log_interval'] == 0:
|
| 408 |
+
# One host transfer per logging interval; hot microbatches remain on CUDA.
|
| 409 |
+
values = (accumulated/logged_steps).cpu().tolist() # lint: ok SYNC-IN-LOOP once per logging interval
|
| 410 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter()-start_time
|
| 411 |
+
emit({'event':'train','step':state['step'],'total_steps':total_steps,
|
| 412 |
+
'epoch_fraction':round(state['epoch']+end/n_train,4),'loss':values[0],
|
| 413 |
+
'classification_loss':values[1],'lm_loss':values[2],'lr':lr,
|
| 414 |
+
'updates_per_second':round(logged_steps/(time.perf_counter()-log_start),3),
|
| 415 |
+
'peak_allocated_gib':round(torch.cuda.max_memory_allocated()/1024**3,3),
|
| 416 |
+
'nonfinite_updates':state['nonfinite_updates'],'elapsed_seconds':round(state['elapsed_seconds'],1)})
|
| 417 |
+
accumulated.zero_(); logged_steps=0;log_start=time.perf_counter()
|
| 418 |
+
if state['step'] % task_config['eval_interval'] == 0:
|
| 419 |
+
evaluate_and_save()
|
| 420 |
+
log_start=time.perf_counter()
|
| 421 |
+
elif state['step'] % config['checkpoint_interval'] == 0:
|
| 422 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter()-start_time
|
| 423 |
+
save('latest.pt')
|
| 424 |
+
if STOP or state['consecutive_nonfinite'] >= config['max_consecutive_nonfinite']:
|
| 425 |
+
stopping_cause = 'user_interrupt' if STOP else 'nonfinite_limit'
|
| 426 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter()-start_time
|
| 427 |
+
save('latest.pt')
|
| 428 |
+
emit({'event':'stopped','stopping_cause':stopping_cause,'step':state['step']})
|
| 429 |
+
log.close()
|
| 430 |
+
return False
|
| 431 |
+
state['epoch'] += 1
|
| 432 |
+
state['offset'] = 0
|
| 433 |
+
order = None
|
| 434 |
+
if state['last_eval_step'] != state['step']:
|
| 435 |
+
evaluate_and_save()
|
| 436 |
+
else:
|
| 437 |
+
save('latest.pt')
|
| 438 |
+
# Final held-out scores use the best validation-selected checkpoint.
|
| 439 |
+
best = torch.load(run_dir/'best.pt',map_location='cpu',weights_only=False,mmap=True)
|
| 440 |
+
model.load_state_dict(best['model'],strict=True)
|
| 441 |
+
best_step = best['state']['step']
|
| 442 |
+
del best
|
| 443 |
+
after = evaluate(model,data[heldout],evaluation_batch)
|
| 444 |
+
(run_dir/'after.json').write_text(json.dumps(after,indent=2),encoding='utf-8')
|
| 445 |
+
rows = raw_rows(task,heldout)
|
| 446 |
+
examples = []
|
| 447 |
+
for row_index,(row,pb,pa) in enumerate(zip(rows,before['predictions'],after['predictions'])):
|
| 448 |
+
example = {'id':f'{task}/{heldout}/{row_index}','input':row.get('text',row.get('article','')),
|
| 449 |
+
'prompt':row.get('question',''),'options':row.get('options',['negative','positive']),
|
| 450 |
+
'gold':row['label'],'predicted_before':pb,'predicted_after':pa,
|
| 451 |
+
'correct_before':pb==row['label'],'correct_after':pa==row['label']}
|
| 452 |
+
examples.append(example)
|
| 453 |
+
(run_dir/'heldout_predictions.json').write_text(json.dumps(examples,ensure_ascii=False,indent=2),encoding='utf-8')
|
| 454 |
+
probe_count = 100 if task=='sst2' else 40
|
| 455 |
+
probe = {'before_correct':sum(e['correct_before'] for e in examples[:probe_count]),
|
| 456 |
+
'after_correct':sum(e['correct_after'] for e in examples[:probe_count]),'total':probe_count}
|
| 457 |
+
state['elapsed_seconds'] = elapsed_prior+time.perf_counter()-start_time
|
| 458 |
+
summary = {'id':task,'name':'SST-2 sentiment' if task=='sst2' else 'RACE middle-school reading',
|
| 459 |
+
'train_examples':n_train,'validation_examples':len(data['validation']['labels']),
|
| 460 |
+
'test_examples':len(data['test']['labels']) if task!='sst2' else 1821,
|
| 461 |
+
'heldout_split':heldout,'baseline':{'label':'Before task training (same classifier)',**full_summary(before)},
|
| 462 |
+
'finetuned':full_summary(after),'chance_accuracy':0.5 if task=='sst2' else 0.25,
|
| 463 |
+
'majority_accuracy':after['majority_accuracy'],'paired_earlier_probe':probe,
|
| 464 |
+
'training':{'epochs':config['epochs'],'max_steps':total_steps,'steps':state['step'],
|
| 465 |
+
'batch_size':config['effective_batch_size'],'microbatch_size':task_config['microbatch_size'],
|
| 466 |
+
'gradient_accumulation':config['effective_batch_size']//task_config['microbatch_size'],
|
| 467 |
+
'learning_rate':config['learning_rate'],'max_length':config['max_length'],
|
| 468 |
+
'lm_coefficient':config['lm_coefficient'],'stopping_cause':stopping_cause,
|
| 469 |
+
'classifier_dropout':config['classifier_dropout'],'native_backbone_dropout':base.config.dropout,
|
| 470 |
+
'nonfinite_updates':state['nonfinite_updates'],'best_step':best_step,
|
| 471 |
+
'peak_allocated_gib':round(torch.cuda.max_memory_allocated()/1024**3,3),
|
| 472 |
+
'trainable_parameters':sum(p.numel() for p in model.parameters()),
|
| 473 |
+
'head':'768-to-2 linear' if task=='sst2' else 'shared 768-to-1 linear over four answers',
|
| 474 |
+
'checkpoint_selection':'highest validation accuracy, lower validation classification loss breaks ties',
|
| 475 |
+
'optimizations':OPTIMIZATIONS,
|
| 476 |
+
'packed_loss_verification':state.get('packed_loss_verification'),
|
| 477 |
+
'details':'Full native backbone trained. Zero decay on oscillator and 1-D parameters. Linear GPT-1 warmup/decay, AdamW, gradient clip1.0. Original backbone dropout retained; classifier dropout0.1. Three unused padding-vocabulary slots initialized for task markers. V2 crops trailing padding using CPU epoch plans and projects only valid auxiliary LM positions; question order, effective batch and candidate-wise loss normalization are preserved.'},
|
| 478 |
+
'examples':examples,'checkpoint':{'name':str((run_dir/'best.pt').relative_to(ROOT)),
|
| 479 |
+
'sha256':digest(run_dir/'best.pt')},
|
| 480 |
+
'elapsed_seconds':state['elapsed_seconds']}
|
| 481 |
+
(run_dir/'summary.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2),encoding='utf-8')
|
| 482 |
+
emit({'event':'complete','step':state['step'],'best_step':best_step,'stopping_cause':stopping_cause,
|
| 483 |
+
'heldout':heldout,'before':full_summary(before),'after':full_summary(after),
|
| 484 |
+
'paired_probe':probe,'elapsed_seconds':round(state['elapsed_seconds'],1)})
|
| 485 |
+
log.close()
|
| 486 |
+
del model,base,optimizer,data
|
| 487 |
+
gc.collect();torch.cuda.empty_cache()
|
| 488 |
+
return True
|
| 489 |
+
|
| 490 |
+
def write_comparison():
|
| 491 |
+
tasks=[]
|
| 492 |
+
for task in ['sst2','race_middle']:
|
| 493 |
+
paths=sorted((ROOT/'runs').glob(f'*_{task}/summary.json'),key=lambda path:path.stat().st_mtime)
|
| 494 |
+
if not paths:
|
| 495 |
+
return
|
| 496 |
+
tasks.append(json.loads(paths[-1].read_text(encoding='utf-8')))
|
| 497 |
+
data={'date':datetime.now(ZoneInfo('Europe/London')).isoformat(),
|
| 498 |
+
'scope':'Full SST-2 training and validation; full RACE-middle training, validation selection and held-out test. Historical GPT-1 figures use different training coverage and test sets.',
|
| 499 |
+
'method':'Two independent full-model adaptations from original pretrained WiggleGPT. Classifier cross-entropy plus0.5 times token-average auxiliary language-model cross-entropy, with GPT-1-style input markers and linear task heads.',
|
| 500 |
+
'control_method':'Identical task inputs and initialized classifier evaluated before task training; after scores use validation-selected trained checkpoint. Earlier language-model likelihood probes are separately identified.',
|
| 501 |
+
'historical':{'gpt1_race_accuracy':59.0,'gpt1_race_middle_accuracy':62.9,'gpt1_sst2_accuracy':91.3,
|
| 502 |
+
'source_url':'https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf',
|
| 503 |
+
'note':'GPT-1 SST-2 91.3% is GLUE test, while this run reports public validation. GPT-1 RACE-M62.9% used combined RACE-middle/high training, while Wiggle used middle-only. Historical rows give context and are not a matched-model replication.'},
|
| 504 |
+
'tasks':tasks,'elapsed_seconds':sum(t['elapsed_seconds'] for t in tasks)}
|
| 505 |
+
(ROOT/'comparison.json').write_text(json.dumps(data,ensure_ascii=False,indent=2),encoding='utf-8')
|
| 506 |
+
|
| 507 |
+
def main():
|
| 508 |
+
parser=argparse.ArgumentParser(description=__doc__)
|
| 509 |
+
parser.add_argument('--task',choices=['all','sst2','race_middle'],default='all')
|
| 510 |
+
parser.add_argument('--resume',type=Path)
|
| 511 |
+
parser.add_argument('--verify-packed-loss-once',action='store_true',
|
| 512 |
+
help='Before the first RACE optimizer update, compare dense and packed auxiliary LM loss on one actual CUDA training candidate and record the result.')
|
| 513 |
+
args=parser.parse_args()
|
| 514 |
+
config=json.loads((ROOT/'train_config.json').read_text(encoding='utf-8'))
|
| 515 |
+
torch.set_num_threads(config['cpu_threads'])
|
| 516 |
+
try:
|
| 517 |
+
import psutil
|
| 518 |
+
psutil.Process().nice(psutil.BELOW_NORMAL_PRIORITY_CLASS)
|
| 519 |
+
except ImportError:
|
| 520 |
+
print('psutil unavailable; continuing with two CPU threads.',flush=True)
|
| 521 |
+
if not torch.cuda.is_available():
|
| 522 |
+
raise RuntimeError('CUDA required; CPU model execution is disabled.')
|
| 523 |
+
torch.cuda.set_device(0)
|
| 524 |
+
torch.set_float32_matmul_precision('high')
|
| 525 |
+
signal.signal(signal.SIGINT,stop_requested)
|
| 526 |
+
if args.resume:
|
| 527 |
+
resume_meta=torch.load(args.resume,map_location='cpu',weights_only=False,mmap=True)
|
| 528 |
+
selected=[resume_meta['task']]
|
| 529 |
+
del resume_meta
|
| 530 |
+
else:
|
| 531 |
+
selected=['sst2','race_middle'] if args.task=='all' else [args.task]
|
| 532 |
+
for task in selected:
|
| 533 |
+
if not run_task(task,config,args.resume,args.verify_packed_loss_once):
|
| 534 |
+
return
|
| 535 |
+
write_comparison()
|
| 536 |
+
print('All requested task adaptations complete.',flush=True)
|
| 537 |
+
|
| 538 |
+
if __name__=='__main__':
|
| 539 |
+
main()
|
task_finetuning/results/train_tasks_v3_original_source.txt
ADDED
|
@@ -0,0 +1,569 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Full-model GPT-1-style task fine-tuning of original WiggleGPT, version 3.
|
| 2 |
+
|
| 3 |
+
Native model_bio.py is imported unchanged. A final-state classifier is added;
|
| 4 |
+
its language-model objective uses the native tied projection. No CPU forwards.
|
| 5 |
+
CPU length plans remove trailing padding; auxiliary LM projection skips padding.
|
| 6 |
+
RACE uses native gradient checkpointing only above 384 cropped tokens.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import argparse
|
| 11 |
+
from collections import Counter
|
| 12 |
+
from datetime import datetime
|
| 13 |
+
from zoneinfo import ZoneInfo
|
| 14 |
+
import gc
|
| 15 |
+
import hashlib
|
| 16 |
+
import json
|
| 17 |
+
import math
|
| 18 |
+
import os
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
import random
|
| 21 |
+
import shutil
|
| 22 |
+
import signal
|
| 23 |
+
import sys
|
| 24 |
+
import time
|
| 25 |
+
|
| 26 |
+
ROOT = Path(__file__).resolve().parent
|
| 27 |
+
WORKSPACE = ROOT.parents[2]
|
| 28 |
+
NATIVE = WORKSPACE / 'Main folders' / 'WiggleGPT v3' / 'WiggleGPT'
|
| 29 |
+
sys.path.insert(0, str(WORKSPACE / 'WiggleGPT-Gosport'))
|
| 30 |
+
sys.path.insert(0, str(NATIVE))
|
| 31 |
+
from wiggle_assistant import load_model
|
| 32 |
+
import model_bio
|
| 33 |
+
import torch
|
| 34 |
+
from torch import nn
|
| 35 |
+
from torch.nn import functional as F
|
| 36 |
+
|
| 37 |
+
STOP = False
|
| 38 |
+
MARKERS = {'start': 50257, 'delimiter': 50258, 'classify': 50259, 'pad': 50256}
|
| 39 |
+
CHECKPOINT_THRESHOLD = 384
|
| 40 |
+
OPTIMIZATIONS = {
|
| 41 |
+
'version': 3,
|
| 42 |
+
'training_padding': 'CPU epoch plan crops each unchanged microbatch to its longest candidate, rounded up to 16 tokens (minimum 16).',
|
| 43 |
+
'auxiliary_lm': 'Project only valid next-token positions; weight each token by the inverse length of its own candidate, then average all candidates.',
|
| 44 |
+
'device_synchronization': 'One order copy to CPU per epoch; CPU-known output_size avoids synchronization when building packed CUDA indices.',
|
| 45 |
+
'question_order_and_effective_batch': 'Original seeded CUDA randperm, effective batch and microbatch membership retained.',
|
| 46 |
+
'native_backbone': 'Unchanged native model_bio.py; RACE checkpointing enabled when cropped width exceeds 384 and disabled at or below 384. SST-2 keeps its configured setting.',
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
def stop_requested(signum, frame):
|
| 50 |
+
global STOP
|
| 51 |
+
STOP = True
|
| 52 |
+
print('Stop requested; finishing the update and saving latest.pt.', flush=True)
|
| 53 |
+
|
| 54 |
+
def digest(path):
|
| 55 |
+
with Path(path).open('rb') as stream:
|
| 56 |
+
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
| 57 |
+
|
| 58 |
+
class TaskModel(nn.Module):
|
| 59 |
+
def __init__(self, base, task, dropout):
|
| 60 |
+
super().__init__()
|
| 61 |
+
self.base = base
|
| 62 |
+
self.task = task
|
| 63 |
+
self.dropout = dropout
|
| 64 |
+
# Added linear classifier is the requested GPT-1 task head, not a backbone replacement.
|
| 65 |
+
with torch.device('cuda:0'):
|
| 66 |
+
self.classifier = nn.Linear(base.config.n_embd, 2 if task == 'sst2' else 1)
|
| 67 |
+
nn.init.normal_(self.classifier.weight, mean=0.0, std=0.02)
|
| 68 |
+
nn.init.zeros_(self.classifier.bias)
|
| 69 |
+
self.hidden = None
|
| 70 |
+
self.last_loss_verification = None
|
| 71 |
+
self.base.transformer.ln_f.register_forward_hook(self.capture_hidden)
|
| 72 |
+
|
| 73 |
+
def capture_hidden(self, module, args, output):
|
| 74 |
+
self.hidden = output
|
| 75 |
+
|
| 76 |
+
def forward(self, tokens, lengths, auxiliary=False, packed_lm_tokens=None,
|
| 77 |
+
verify_packed=False):
|
| 78 |
+
batch, choices, width = tokens.shape
|
| 79 |
+
flat = tokens.reshape(batch * choices, width)
|
| 80 |
+
self.base(flat)
|
| 81 |
+
hidden = self.hidden
|
| 82 |
+
self.hidden = None
|
| 83 |
+
positions = lengths.reshape(-1) - 1
|
| 84 |
+
features = hidden[torch.arange(batch * choices, device=flat.device), positions]
|
| 85 |
+
features = features.reshape(batch, choices, -1)
|
| 86 |
+
# GPT-1's multiple-choice head shares a dropout feature mask across options.
|
| 87 |
+
if self.training and self.dropout:
|
| 88 |
+
mask = F.dropout(torch.ones((batch, 1, features.shape[-1]), device=features.device,
|
| 89 |
+
dtype=features.dtype), p=self.dropout, training=True)
|
| 90 |
+
features = features * mask
|
| 91 |
+
scores = self.classifier(features).squeeze(1) if self.task == 'sst2' else self.classifier(features).squeeze(-1)
|
| 92 |
+
lm_loss = None
|
| 93 |
+
if auxiliary:
|
| 94 |
+
# Average within each candidate sequence, then across all candidates.
|
| 95 |
+
# output_size is supplied by the CPU length plan, avoiding a CUDA
|
| 96 |
+
# synchronization to discover the size of a boolean-index result.
|
| 97 |
+
if packed_lm_tokens is None:
|
| 98 |
+
raise ValueError('Auxiliary LM requires the CPU-planned valid-token count.')
|
| 99 |
+
counts = lengths.reshape(-1) - 1
|
| 100 |
+
sequence_ids = torch.repeat_interleave(
|
| 101 |
+
torch.arange(batch * choices, device=flat.device), counts,
|
| 102 |
+
output_size=packed_lm_tokens)
|
| 103 |
+
starts = counts.cumsum(0) - counts
|
| 104 |
+
token_positions = (torch.arange(packed_lm_tokens, device=flat.device)
|
| 105 |
+
- starts[sequence_ids])
|
| 106 |
+
flat_positions = sequence_ids * width + token_positions
|
| 107 |
+
lm_inputs = hidden.reshape(-1, hidden.shape[-1]).index_select(0, flat_positions)
|
| 108 |
+
targets = flat.reshape(-1).index_select(0, flat_positions + 1).long()
|
| 109 |
+
logits = self.base.lm_head(lm_inputs)
|
| 110 |
+
per_token = F.cross_entropy(logits.float(), targets, reduction='none')
|
| 111 |
+
lm_loss = (per_token / counts[sequence_ids]).sum() / (batch * choices)
|
| 112 |
+
if verify_packed:
|
| 113 |
+
# Both paths use these same actual CUDA hidden states, tied
|
| 114 |
+
# projection, autocast dtype and per-candidate normalization.
|
| 115 |
+
# Limit the comparison to one real candidate to keep the
|
| 116 |
+
# optional once-only verification below the desktop VRAM cap.
|
| 117 |
+
with torch.no_grad():
|
| 118 |
+
dense_logits = self.base.lm_head(hidden[:1, :-1].detach())
|
| 119 |
+
dense_targets = flat[:1, 1:].long()
|
| 120 |
+
dense_tokens = F.cross_entropy(
|
| 121 |
+
dense_logits.float().reshape(-1, dense_logits.shape[-1]),
|
| 122 |
+
dense_targets.reshape(-1), reduction='none').view_as(dense_targets)
|
| 123 |
+
dense_mask = torch.arange(width - 1, device=flat.device)[None, :] < counts[:1, None]
|
| 124 |
+
dense_loss = ((dense_tokens * dense_mask).sum(-1)
|
| 125 |
+
/ counts[:1].clamp_min(1)).mean()
|
| 126 |
+
packed_first = ((per_token.detach() / counts[sequence_ids])
|
| 127 |
+
* (sequence_ids == 0)).sum()
|
| 128 |
+
values = torch.stack((dense_loss, packed_first,
|
| 129 |
+
(dense_loss - packed_first).abs())).cpu().tolist() # lint: ok COMPLEX-ABS real scalar CE losses
|
| 130 |
+
self.last_loss_verification = {
|
| 131 |
+
'passed': math.isclose(values[0], values[1], rel_tol=1e-4, abs_tol=1e-5),
|
| 132 |
+
'dense_loss': values[0], 'packed_loss': values[1],
|
| 133 |
+
'absolute_difference': values[2], 'relative_tolerance': 1e-4,
|
| 134 |
+
'absolute_tolerance': 1e-5, 'candidates_checked': 1,
|
| 135 |
+
'actual_training_candidates': batch * choices,
|
| 136 |
+
'packed_lm_tokens': packed_lm_tokens, 'cropped_width': width,
|
| 137 |
+
'projection_dtype': str(logits.dtype),
|
| 138 |
+
'device': str(hidden.device),
|
| 139 |
+
'comparison': 'Same actual training hidden states; dense masked CE versus packed valid-position CE, before the first optimizer update.',
|
| 140 |
+
}
|
| 141 |
+
return scores.float(), lm_loss
|
| 142 |
+
|
| 143 |
+
def load_data(task, device):
|
| 144 |
+
prefix = 'sst2' if task == 'sst2' else 'race_middle'
|
| 145 |
+
splits = ['train', 'validation'] + (['test'] if task != 'sst2' else [])
|
| 146 |
+
data = {}
|
| 147 |
+
for split in splits:
|
| 148 |
+
path = ROOT / 'data' / f'{prefix}_{split}_encoded.pt'
|
| 149 |
+
if not path.exists():
|
| 150 |
+
# Preparation script may use the compact .pt suffix.
|
| 151 |
+
path = ROOT / 'data' / f'{prefix}_{split}.pt'
|
| 152 |
+
cache = torch.load(path, map_location='cpu', weights_only=False, mmap=True)
|
| 153 |
+
# Both conventional names are accepted; contents stay unchanged.
|
| 154 |
+
ids = cache.get('input_ids', cache.get('tokens'))
|
| 155 |
+
data[split] = {'input_ids': ids.to(device), 'lengths': cache['lengths'].to(device),
|
| 156 |
+
'lengths_cpu': cache['lengths'].clone(),
|
| 157 |
+
'labels': cache['labels'].to(device), 'cache_path': str(path),
|
| 158 |
+
'metadata': {k:v for k,v in cache.items() if not torch.is_tensor(v)}}
|
| 159 |
+
return data
|
| 160 |
+
|
| 161 |
+
def select_batch(data, index, width=None):
|
| 162 |
+
tokens = data['input_ids'][index]
|
| 163 |
+
if width is not None:
|
| 164 |
+
tokens = tokens[..., :width]
|
| 165 |
+
return tokens, data['lengths'][index], data['labels'][index]
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def plan_epoch_microbatches(data, order, effective_batch_size, microbatch_size):
|
| 169 |
+
"""Plan CPU scalar lengths once, preserving the saved CUDA question order."""
|
| 170 |
+
ordered_lengths = data['lengths_cpu'].index_select(0, order.detach().cpu())
|
| 171 |
+
cached_width = data['input_ids'].shape[-1]
|
| 172 |
+
plan = {}
|
| 173 |
+
for start in range(0, len(order), effective_batch_size):
|
| 174 |
+
end = min(len(order), start + effective_batch_size)
|
| 175 |
+
for position in range(start, end, microbatch_size):
|
| 176 |
+
lengths = ordered_lengths[position:min(end, position + microbatch_size)]
|
| 177 |
+
width = min(cached_width, max(16, math.ceil(int(lengths.max()) / 16) * 16))
|
| 178 |
+
plan[position] = (width, int((lengths - 1).sum()))
|
| 179 |
+
return plan
|
| 180 |
+
|
| 181 |
+
def evaluate(model, data, batch_size, fixed_indices=None):
|
| 182 |
+
model.eval()
|
| 183 |
+
predictions, losses = [], []
|
| 184 |
+
if fixed_indices is None:
|
| 185 |
+
fixed_indices = torch.arange(data['labels'].shape[0], device='cuda:0')
|
| 186 |
+
# AMP-COMPLEX: native weights/activations are real; native RoPE explicitly casts its complex rotation to float32.
|
| 187 |
+
with torch.inference_mode(), torch.autocast('cuda', dtype=torch.bfloat16): # lint: ok AMP-COMPLEX real native weights; RoPE explicitly uses float32 complex rotation
|
| 188 |
+
for start in range(0, len(fixed_indices), batch_size):
|
| 189 |
+
indices = fixed_indices[start:start + batch_size]
|
| 190 |
+
x, lengths, labels = select_batch(data, indices)
|
| 191 |
+
logits, _ = model(x, lengths)
|
| 192 |
+
predictions.append(logits.argmax(-1))
|
| 193 |
+
losses.append(F.cross_entropy(logits, labels, reduction='sum'))
|
| 194 |
+
predicted = torch.cat(predictions)
|
| 195 |
+
labels = data['labels'][fixed_indices]
|
| 196 |
+
correct = int((predicted == labels).sum())
|
| 197 |
+
counts = torch.bincount(labels, minlength=2 if model.task == 'sst2' else 4).cpu().tolist()
|
| 198 |
+
return {'correct': correct, 'total': len(labels), 'accuracy': correct / len(labels),
|
| 199 |
+
'loss': float(torch.stack(losses).sum() / len(labels)),
|
| 200 |
+
'predictions': predicted.cpu().tolist(), 'gold_label_counts': counts,
|
| 201 |
+
'majority_accuracy': max(counts) / len(labels)}
|
| 202 |
+
|
| 203 |
+
def optimizer_for(model, config):
|
| 204 |
+
decay, no_decay = [], []
|
| 205 |
+
protected = ('omega', 'phi', 'baseline', 'norm', 'bias', 'scale', 'phase')
|
| 206 |
+
for name, parameter in model.named_parameters():
|
| 207 |
+
if parameter.ndim >= 2 and not parameter.is_complex() and not any(word in name for word in protected):
|
| 208 |
+
decay.append(parameter)
|
| 209 |
+
else:
|
| 210 |
+
no_decay.append(parameter)
|
| 211 |
+
print(f'Decay: {sum(p.numel() for p in decay):,}; protected/no decay: {sum(p.numel() for p in no_decay):,}', flush=True)
|
| 212 |
+
return torch.optim.AdamW([{'params':decay, 'weight_decay':config['weight_decay']},
|
| 213 |
+
{'params':no_decay, 'weight_decay':0.0}],
|
| 214 |
+
lr=config['learning_rate'], betas=tuple(config['betas']), eps=config['epsilon'], foreach=True)
|
| 215 |
+
|
| 216 |
+
def schedule(step, total, config):
|
| 217 |
+
warmup = max(1, round(total * config['warmup_fraction']))
|
| 218 |
+
if step < warmup:
|
| 219 |
+
return config['learning_rate'] * (step + 1) / warmup
|
| 220 |
+
return config['learning_rate'] * max(0.0, (total - step) / max(1, total - warmup))
|
| 221 |
+
|
| 222 |
+
def atomic_save(path, payload):
|
| 223 |
+
temporary = path.with_suffix('.pending.pt')
|
| 224 |
+
torch.save(payload, temporary)
|
| 225 |
+
temporary.replace(path)
|
| 226 |
+
|
| 227 |
+
def rng_state(generator):
|
| 228 |
+
return {'python': random.getstate(), 'torch': torch.get_rng_state(),
|
| 229 |
+
'cuda': torch.cuda.get_rng_state(0), 'batch_generator': generator.get_state()}
|
| 230 |
+
|
| 231 |
+
def restore_rng(state, generator):
|
| 232 |
+
random.setstate(state['python'])
|
| 233 |
+
torch.set_rng_state(state['torch'].cpu())
|
| 234 |
+
torch.cuda.set_rng_state(state['cuda'].cpu(), 0)
|
| 235 |
+
generator.set_state(state['batch_generator'].cpu())
|
| 236 |
+
|
| 237 |
+
def full_summary(result):
|
| 238 |
+
return {k:v for k,v in result.items() if k != 'predictions'}
|
| 239 |
+
|
| 240 |
+
def raw_rows(task, split):
|
| 241 |
+
path = ROOT / 'data' / f'{task}_{split}.jsonl'
|
| 242 |
+
return [json.loads(line) for line in path.read_text(encoding='utf-8').splitlines()]
|
| 243 |
+
|
| 244 |
+
def run_task(task, config, resume=None, verify_packed_loss_once=False):
|
| 245 |
+
task_config = config['tasks'][task]
|
| 246 |
+
random.seed(config['seed'])
|
| 247 |
+
torch.manual_seed(config['seed'])
|
| 248 |
+
torch.cuda.manual_seed_all(config['seed'])
|
| 249 |
+
base, native_config, encoder, metadata = load_model((ROOT / config['starting_checkpoint']).resolve())
|
| 250 |
+
free, total = torch.cuda.mem_get_info()
|
| 251 |
+
cap = min(int(config['max_process_gib'] * 1024**3),
|
| 252 |
+
total - int(config['desktop_reserve_gib'] * 1024**3),
|
| 253 |
+
free + torch.cuda.memory_reserved() - int(0.5 * 1024**3))
|
| 254 |
+
torch.cuda.set_per_process_memory_fraction(cap / total, 0)
|
| 255 |
+
base.config.gradient_checkpointing = task_config['gradient_checkpointing']
|
| 256 |
+
metadata['checkpoint_sha256'] = digest((ROOT / config['starting_checkpoint']).resolve())
|
| 257 |
+
model = TaskModel(base, task, config['classifier_dropout'])
|
| 258 |
+
# Reuse three existing padding slots; real GPT-2 IDs and tokenizer are unchanged.
|
| 259 |
+
with torch.no_grad():
|
| 260 |
+
marker_ids = torch.tensor([MARKERS['start'],MARKERS['delimiter'],MARKERS['classify']], device='cuda:0')
|
| 261 |
+
base.transformer.wte.weight.index_copy_(0, marker_ids,
|
| 262 |
+
torch.randn((3, base.config.n_embd), device='cuda:0') * 0.02)
|
| 263 |
+
optimizer = optimizer_for(model, config)
|
| 264 |
+
data = load_data(task, 'cuda:0')
|
| 265 |
+
n_train = data['train']['labels'].shape[0]
|
| 266 |
+
per_epoch = math.ceil(n_train / config['effective_batch_size'])
|
| 267 |
+
total_steps = per_epoch * config['epochs']
|
| 268 |
+
generator = torch.Generator(device='cuda:0').manual_seed(config['seed'] + 123)
|
| 269 |
+
if resume:
|
| 270 |
+
checkpoint = torch.load(resume, map_location='cpu', weights_only=False, mmap=True)
|
| 271 |
+
model.load_state_dict(checkpoint['model'], strict=True)
|
| 272 |
+
optimizer.load_state_dict(checkpoint['optimizer'])
|
| 273 |
+
state = checkpoint['state']
|
| 274 |
+
order = checkpoint['order'].to('cuda:0') if checkpoint['order'] is not None else None
|
| 275 |
+
restore_rng(checkpoint['rng'], generator)
|
| 276 |
+
run_dir = Path(resume).resolve().parent
|
| 277 |
+
del checkpoint
|
| 278 |
+
# Preserve this continuation's actual source/config alongside the
|
| 279 |
+
# original run, without overwriting its v2 source or provenance.
|
| 280 |
+
segment_dir = (run_dir / 'resume_segments'
|
| 281 |
+
/ f'{datetime.now(ZoneInfo("Europe/London")).strftime("%Y-%m-%d_%H-%M-%S_%f")}_{Path(__file__).stem}')
|
| 282 |
+
segment_dir.mkdir(parents=True, exist_ok=False)
|
| 283 |
+
shutil.copy2(__file__, segment_dir / Path(__file__).name)
|
| 284 |
+
shutil.copy2(model_bio.__file__, segment_dir / 'model_bio.py')
|
| 285 |
+
(segment_dir/'config.json').write_text(json.dumps(config,indent=2),encoding='utf-8')
|
| 286 |
+
continuation = {
|
| 287 |
+
'resume_checkpoint':str(Path(resume).resolve()),
|
| 288 |
+
'resume_checkpoint_sha256':digest(resume),
|
| 289 |
+
'start_step':state['step'],'epoch':state['epoch'],'offset':state['offset'],
|
| 290 |
+
'trainer_source':Path(__file__).name,'trainer_sha256':digest(__file__),
|
| 291 |
+
'model_source_sha256':digest(model_bio.__file__),
|
| 292 |
+
'configuration_source':'Saved resume checkpoint configuration',
|
| 293 |
+
'optimizations':OPTIMIZATIONS,
|
| 294 |
+
}
|
| 295 |
+
(segment_dir/'provenance.json').write_text(json.dumps(continuation,indent=2),encoding='utf-8')
|
| 296 |
+
state.setdefault('resume_segments',[]).append(str(segment_dir.relative_to(run_dir)))
|
| 297 |
+
else:
|
| 298 |
+
run_dir = ROOT / 'runs' / f'{Path(__file__).stem}_{datetime.now(ZoneInfo("Europe/London")).strftime("%Y-%m-%d_%H-%M-%S")}_{task}'
|
| 299 |
+
run_dir.mkdir(parents=True, exist_ok=False)
|
| 300 |
+
state = {'step':0, 'epoch':0, 'offset':0, 'best_accuracy':-1.0, 'best_loss':float('inf'),
|
| 301 |
+
'best_step':0, 'nonfinite_updates':0, 'consecutive_nonfinite':0, 'last_eval_step':-1,
|
| 302 |
+
'elapsed_seconds':0.0, 'lr_scale':1.0, 'packed_loss_verification':None}
|
| 303 |
+
order = None
|
| 304 |
+
(run_dir / 'config.json').write_text(json.dumps(config, indent=2), encoding='utf-8')
|
| 305 |
+
shutil.copy2(__file__, run_dir / Path(__file__).name)
|
| 306 |
+
shutil.copy2(model_bio.__file__, run_dir / 'model_bio.py')
|
| 307 |
+
provenance = {'model_source_sha256':digest(model_bio.__file__), 'trainer_sha256':digest(__file__),
|
| 308 |
+
'starting_checkpoint':metadata, 'torch':torch.__version__, 'markers':MARKERS,
|
| 309 |
+
'native_backbone_dropout':base.config.dropout,
|
| 310 |
+
'data':{split:d['metadata'] for split,d in data.items()},
|
| 311 |
+
'all_backbone_parameters_train':True, 'optimizations':OPTIMIZATIONS}
|
| 312 |
+
(run_dir/'provenance.json').write_text(json.dumps(provenance, indent=2), encoding='utf-8')
|
| 313 |
+
state.setdefault('adaptive_checkpointing_microbatches',{'enabled':0,'disabled':0})
|
| 314 |
+
log = (run_dir / (run_dir.name + '.log')).open('a', encoding='utf-8', buffering=1)
|
| 315 |
+
def emit(event):
|
| 316 |
+
event.update(task=task)
|
| 317 |
+
line = json.dumps(event, ensure_ascii=False)
|
| 318 |
+
print(line, flush=True)
|
| 319 |
+
log.write(line + '\n')
|
| 320 |
+
with (run_dir/'metrics.jsonl').open('a',encoding='utf-8') as stream:
|
| 321 |
+
stream.write(line + '\n')
|
| 322 |
+
def save(name):
|
| 323 |
+
atomic_save(run_dir/name, {'model':model.state_dict(), 'optimizer':optimizer.state_dict(),
|
| 324 |
+
'state':dict(state), 'order':order, 'rng':rng_state(generator), 'config':config,
|
| 325 |
+
'task':task, 'model_args':vars(base.config), 'markers':MARKERS,
|
| 326 |
+
'source_hashes':{'model':digest(model_bio.__file__),'trainer':digest(__file__)}})
|
| 327 |
+
evaluation_batch = 16 if task == 'sst2' else 4
|
| 328 |
+
heldout = 'validation' if task == 'sst2' else 'test'
|
| 329 |
+
before_path = run_dir/'before.json'
|
| 330 |
+
start_time = time.perf_counter()
|
| 331 |
+
elapsed_prior = state['elapsed_seconds']
|
| 332 |
+
emit({'event':'start','step':state['step'],'train_examples':n_train,'epochs':config['epochs'],
|
| 333 |
+
'total_steps':total_steps,'microbatch':task_config['microbatch_size'],
|
| 334 |
+
'effective_batch':config['effective_batch_size'],'memory_cap_gib':round(cap/1024**3,2),
|
| 335 |
+
'trainable_parameters':sum(p.numel() for p in model.parameters() if p.requires_grad),
|
| 336 |
+
'device':torch.cuda.get_device_name(0), 'optimizations':OPTIMIZATIONS,
|
| 337 |
+
'verify_packed_loss_once':verify_packed_loss_once})
|
| 338 |
+
if not before_path.exists():
|
| 339 |
+
before = evaluate(model, data[heldout], evaluation_batch)
|
| 340 |
+
before_path.write_text(json.dumps(before,indent=2),encoding='utf-8')
|
| 341 |
+
emit({'event':'before','heldout':heldout,**full_summary(before)})
|
| 342 |
+
else:
|
| 343 |
+
before = json.loads(before_path.read_text(encoding='utf-8'))
|
| 344 |
+
train_fixed = torch.arange(min(128,n_train), device='cuda:0')
|
| 345 |
+
def evaluate_and_save():
|
| 346 |
+
validation = evaluate(model, data['validation'], evaluation_batch)
|
| 347 |
+
train_eval = evaluate(model, data['train'], evaluation_batch, train_fixed)
|
| 348 |
+
changed = (validation['accuracy'] > state['best_accuracy'] or
|
| 349 |
+
(validation['accuracy'] == state['best_accuracy'] and validation['loss'] < state['best_loss']))
|
| 350 |
+
if changed:
|
| 351 |
+
state['best_accuracy'] = validation['accuracy']
|
| 352 |
+
state['best_loss'] = validation['loss']
|
| 353 |
+
state['best_step'] = state['step']
|
| 354 |
+
state['last_eval_step'] = state['step']
|
| 355 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter() - start_time
|
| 356 |
+
# Once-per-evaluation diagnostic over twelve oscillator tensors, not Python-built training batches.
|
| 357 |
+
omegas = torch.cat([p.detach().reshape(-1) for name,p in model.named_parameters() if name.endswith('.omega')]) # lint: ok STACK-COMPREHENSION once-per-evaluation statistics over twelve oscillator tensors
|
| 358 |
+
emit({'event':'eval','step':state['step'],'epoch':state['epoch'],'offset':state['offset'],
|
| 359 |
+
'train':full_summary(train_eval),'validation':full_summary(validation),
|
| 360 |
+
'best_step':state['best_step'],'omega_std':float(omegas.std(correction=0)),
|
| 361 |
+
'peak_allocated_gib':round(torch.cuda.max_memory_allocated()/1024**3,3),
|
| 362 |
+
'sample':{'gold':int(data['validation']['labels'][0]),'predicted':validation['predictions'][0]},
|
| 363 |
+
'elapsed_seconds':round(state['elapsed_seconds'],1)})
|
| 364 |
+
if changed:
|
| 365 |
+
save('best.pt')
|
| 366 |
+
save('latest.pt')
|
| 367 |
+
model.train()
|
| 368 |
+
if state['last_eval_step'] < 0:
|
| 369 |
+
evaluate_and_save()
|
| 370 |
+
model.train()
|
| 371 |
+
accumulated = torch.zeros(3,device='cuda:0')
|
| 372 |
+
logged_steps = 0
|
| 373 |
+
log_start = time.perf_counter()
|
| 374 |
+
stopping_cause = 'epochs_complete'
|
| 375 |
+
while state['epoch'] < config['epochs']:
|
| 376 |
+
if order is None:
|
| 377 |
+
order = torch.randperm(n_train,device='cuda:0',generator=generator)
|
| 378 |
+
# Reconstructed identically from saved order/CPU lengths after resume;
|
| 379 |
+
# no extra RNG draws and no per-microbatch CUDA-to-CPU length transfers.
|
| 380 |
+
microbatch_plan = plan_epoch_microbatches(
|
| 381 |
+
data['train'], order, config['effective_batch_size'], task_config['microbatch_size'])
|
| 382 |
+
while state['offset'] < n_train:
|
| 383 |
+
end = min(n_train,state['offset']+config['effective_batch_size'])
|
| 384 |
+
indices = order[state['offset']:end]
|
| 385 |
+
effective = len(indices)
|
| 386 |
+
lr = schedule(state['step'], total_steps, config) * state['lr_scale']
|
| 387 |
+
for group in optimizer.param_groups:
|
| 388 |
+
group['lr'] = lr
|
| 389 |
+
optimizer.zero_grad(set_to_none=True)
|
| 390 |
+
batch_values = torch.zeros(3,device='cuda:0')
|
| 391 |
+
for position in range(0,effective,task_config['microbatch_size']):
|
| 392 |
+
micro = indices[position:position+task_config['microbatch_size']]
|
| 393 |
+
width, packed_lm_tokens = microbatch_plan[state['offset'] + position]
|
| 394 |
+
x,lengths,labels = select_batch(data['train'],micro,width)
|
| 395 |
+
if task != 'sst2':
|
| 396 |
+
base.config.gradient_checkpointing = width > CHECKPOINT_THRESHOLD
|
| 397 |
+
checkpoint_key = 'enabled' if base.config.gradient_checkpointing else 'disabled'
|
| 398 |
+
state['adaptive_checkpointing_microbatches'][checkpoint_key] += 1
|
| 399 |
+
verify_this = (verify_packed_loss_once and task != 'sst2'
|
| 400 |
+
and not state.get('packed_loss_verification'))
|
| 401 |
+
# AMP-COMPLEX: real-valued model; native rotary complex operations remain explicitly float32.
|
| 402 |
+
with torch.autocast('cuda',dtype=torch.bfloat16): # lint: ok AMP-COMPLEX real native weights; RoPE explicitly uses float32 complex rotation
|
| 403 |
+
logits,lm_loss = model(x,lengths,auxiliary=True,
|
| 404 |
+
packed_lm_tokens=packed_lm_tokens,verify_packed=verify_this)
|
| 405 |
+
classification_loss = F.cross_entropy(logits,labels)
|
| 406 |
+
loss = classification_loss + config['lm_coefficient'] * lm_loss
|
| 407 |
+
weighted = loss * (len(micro)/effective)
|
| 408 |
+
if verify_this:
|
| 409 |
+
verification = model.last_loss_verification
|
| 410 |
+
state['packed_loss_verification'] = verification
|
| 411 |
+
(run_dir/'packed_loss_verification.json').write_text(
|
| 412 |
+
json.dumps(verification,indent=2),encoding='utf-8')
|
| 413 |
+
emit({'event':'packed_loss_verification','step':state['step'],**verification})
|
| 414 |
+
if not verification['passed']:
|
| 415 |
+
raise RuntimeError('Packed auxiliary LM loss did not match dense masked loss; no optimizer update made.')
|
| 416 |
+
weighted.backward()
|
| 417 |
+
batch_values += torch.stack((loss.detach(),classification_loss.detach(),lm_loss.detach())) * (len(micro)/effective)
|
| 418 |
+
norm = torch.nn.utils.clip_grad_norm_(model.parameters(),config['grad_clip'])
|
| 419 |
+
if torch.isfinite(norm):
|
| 420 |
+
optimizer.step()
|
| 421 |
+
state['consecutive_nonfinite'] = 0
|
| 422 |
+
else:
|
| 423 |
+
state['nonfinite_updates'] += 1
|
| 424 |
+
state['consecutive_nonfinite'] += 1
|
| 425 |
+
if state['consecutive_nonfinite'] >= 2:
|
| 426 |
+
state['lr_scale'] *= 0.5
|
| 427 |
+
emit({'event':'nonfinite_update_skipped','step':state['step']+1,
|
| 428 |
+
'count':state['nonfinite_updates'],'lr_scale':state['lr_scale']})
|
| 429 |
+
state['step'] += 1
|
| 430 |
+
state['offset'] = end
|
| 431 |
+
accumulated += batch_values
|
| 432 |
+
logged_steps += 1
|
| 433 |
+
if state['step'] == 1 or state['step'] % config['log_interval'] == 0:
|
| 434 |
+
# One host transfer per logging interval; hot microbatches remain on CUDA.
|
| 435 |
+
values = (accumulated/logged_steps).cpu().tolist() # lint: ok SYNC-IN-LOOP once per logging interval
|
| 436 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter()-start_time
|
| 437 |
+
emit({'event':'train','step':state['step'],'total_steps':total_steps,
|
| 438 |
+
'epoch_fraction':round(state['epoch']+end/n_train,4),'loss':values[0],
|
| 439 |
+
'classification_loss':values[1],'lm_loss':values[2],'lr':lr,
|
| 440 |
+
'updates_per_second':round(logged_steps/(time.perf_counter()-log_start),3),
|
| 441 |
+
'peak_allocated_gib':round(torch.cuda.max_memory_allocated()/1024**3,3),
|
| 442 |
+
'nonfinite_updates':state['nonfinite_updates'],'elapsed_seconds':round(state['elapsed_seconds'],1)})
|
| 443 |
+
accumulated.zero_(); logged_steps=0;log_start=time.perf_counter()
|
| 444 |
+
if state['step'] % task_config['eval_interval'] == 0:
|
| 445 |
+
evaluate_and_save()
|
| 446 |
+
log_start=time.perf_counter()
|
| 447 |
+
elif state['step'] % config['checkpoint_interval'] == 0:
|
| 448 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter()-start_time
|
| 449 |
+
save('latest.pt')
|
| 450 |
+
if STOP or state['consecutive_nonfinite'] >= config['max_consecutive_nonfinite']:
|
| 451 |
+
stopping_cause = 'user_interrupt' if STOP else 'nonfinite_limit'
|
| 452 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter()-start_time
|
| 453 |
+
save('latest.pt')
|
| 454 |
+
emit({'event':'stopped','stopping_cause':stopping_cause,'step':state['step']})
|
| 455 |
+
log.close()
|
| 456 |
+
return False
|
| 457 |
+
state['epoch'] += 1
|
| 458 |
+
state['offset'] = 0
|
| 459 |
+
order = None
|
| 460 |
+
if state['last_eval_step'] != state['step']:
|
| 461 |
+
evaluate_and_save()
|
| 462 |
+
else:
|
| 463 |
+
save('latest.pt')
|
| 464 |
+
# Final held-out scores use the best validation-selected checkpoint.
|
| 465 |
+
best = torch.load(run_dir/'best.pt',map_location='cpu',weights_only=False,mmap=True)
|
| 466 |
+
model.load_state_dict(best['model'],strict=True)
|
| 467 |
+
best_step = best['state']['step']
|
| 468 |
+
del best
|
| 469 |
+
after = evaluate(model,data[heldout],evaluation_batch)
|
| 470 |
+
(run_dir/'after.json').write_text(json.dumps(after,indent=2),encoding='utf-8')
|
| 471 |
+
rows = raw_rows(task,heldout)
|
| 472 |
+
examples = []
|
| 473 |
+
for row_index,(row,pb,pa) in enumerate(zip(rows,before['predictions'],after['predictions'])):
|
| 474 |
+
example = {'id':f'{task}/{heldout}/{row_index}','input':row.get('text',row.get('article','')),
|
| 475 |
+
'prompt':row.get('question',''),'options':row.get('options',['negative','positive']),
|
| 476 |
+
'gold':row['label'],'predicted_before':pb,'predicted_after':pa,
|
| 477 |
+
'correct_before':pb==row['label'],'correct_after':pa==row['label']}
|
| 478 |
+
examples.append(example)
|
| 479 |
+
(run_dir/'heldout_predictions.json').write_text(json.dumps(examples,ensure_ascii=False,indent=2),encoding='utf-8')
|
| 480 |
+
probe_count = 100 if task=='sst2' else 40
|
| 481 |
+
probe = {'before_correct':sum(e['correct_before'] for e in examples[:probe_count]),
|
| 482 |
+
'after_correct':sum(e['correct_after'] for e in examples[:probe_count]),'total':probe_count}
|
| 483 |
+
state['elapsed_seconds'] = elapsed_prior+time.perf_counter()-start_time
|
| 484 |
+
summary = {'id':task,'name':'SST-2 sentiment' if task=='sst2' else 'RACE middle-school reading',
|
| 485 |
+
'train_examples':n_train,'validation_examples':len(data['validation']['labels']),
|
| 486 |
+
'test_examples':len(data['test']['labels']) if task!='sst2' else 1821,
|
| 487 |
+
'heldout_split':heldout,'baseline':{'label':'Before task training (same classifier)',**full_summary(before)},
|
| 488 |
+
'finetuned':full_summary(after),'chance_accuracy':0.5 if task=='sst2' else 0.25,
|
| 489 |
+
'majority_accuracy':after['majority_accuracy'],'paired_earlier_probe':probe,
|
| 490 |
+
'training':{'epochs':config['epochs'],'max_steps':total_steps,'steps':state['step'],
|
| 491 |
+
'batch_size':config['effective_batch_size'],'microbatch_size':task_config['microbatch_size'],
|
| 492 |
+
'gradient_accumulation':config['effective_batch_size']//task_config['microbatch_size'],
|
| 493 |
+
'learning_rate':config['learning_rate'],'max_length':config['max_length'],
|
| 494 |
+
'lm_coefficient':config['lm_coefficient'],'stopping_cause':stopping_cause,
|
| 495 |
+
'classifier_dropout':config['classifier_dropout'],'native_backbone_dropout':base.config.dropout,
|
| 496 |
+
'nonfinite_updates':state['nonfinite_updates'],'best_step':best_step,
|
| 497 |
+
'peak_allocated_gib':round(torch.cuda.max_memory_allocated()/1024**3,3),
|
| 498 |
+
'trainable_parameters':sum(p.numel() for p in model.parameters()),
|
| 499 |
+
'head':'768-to-2 linear' if task=='sst2' else 'shared 768-to-1 linear over four answers',
|
| 500 |
+
'checkpoint_selection':'highest validation accuracy, lower validation classification loss breaks ties',
|
| 501 |
+
'optimizations':OPTIMIZATIONS,
|
| 502 |
+
'adaptive_checkpointing_microbatches':state['adaptive_checkpointing_microbatches'],
|
| 503 |
+
'resume_segments':state.get('resume_segments',[]),
|
| 504 |
+
'packed_loss_verification':state.get('packed_loss_verification'),
|
| 505 |
+
'details':'Full native backbone trained. Zero decay on oscillator and 1-D parameters. Linear GPT-1 warmup/decay, AdamW, gradient clip1.0. Original backbone dropout retained; classifier dropout0.1. Three unused padding-vocabulary slots initialized for task markers. V2 padding cropping and packed LM projection retained. V3 RACE native gradient checkpointing is enabled above 384 cropped tokens and disabled at or below 384; counters cover v3 segments, while earlier v2 segments used checkpointing throughout. Question order, effective batch and candidate-wise loss normalization are preserved.'},
|
| 506 |
+
'examples':examples,'checkpoint':{'name':str((run_dir/'best.pt').relative_to(ROOT)),
|
| 507 |
+
'sha256':digest(run_dir/'best.pt')},
|
| 508 |
+
'elapsed_seconds':state['elapsed_seconds']}
|
| 509 |
+
(run_dir/'summary.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2),encoding='utf-8')
|
| 510 |
+
emit({'event':'complete','step':state['step'],'best_step':best_step,'stopping_cause':stopping_cause,
|
| 511 |
+
'heldout':heldout,'before':full_summary(before),'after':full_summary(after),
|
| 512 |
+
'paired_probe':probe,'elapsed_seconds':round(state['elapsed_seconds'],1)})
|
| 513 |
+
log.close()
|
| 514 |
+
del model,base,optimizer,data
|
| 515 |
+
gc.collect();torch.cuda.empty_cache()
|
| 516 |
+
return True
|
| 517 |
+
|
| 518 |
+
def write_comparison():
|
| 519 |
+
tasks=[]
|
| 520 |
+
for task in ['sst2','race_middle']:
|
| 521 |
+
paths=sorted((ROOT/'runs').glob(f'*_{task}/summary.json'),key=lambda path:path.stat().st_mtime)
|
| 522 |
+
if not paths:
|
| 523 |
+
return
|
| 524 |
+
tasks.append(json.loads(paths[-1].read_text(encoding='utf-8')))
|
| 525 |
+
data={'date':datetime.now(ZoneInfo('Europe/London')).isoformat(),
|
| 526 |
+
'scope':'Full SST-2 training and validation; full RACE-middle training, validation selection and held-out test. Historical GPT-1 figures use different training coverage and test sets.',
|
| 527 |
+
'method':'Two independent full-model adaptations from original pretrained WiggleGPT. Classifier cross-entropy plus0.5 times token-average auxiliary language-model cross-entropy, with GPT-1-style input markers and linear task heads.',
|
| 528 |
+
'control_method':'Identical task inputs and initialized classifier evaluated before task training; after scores use validation-selected trained checkpoint. Earlier language-model likelihood probes are separately identified.',
|
| 529 |
+
'historical':{'gpt1_race_accuracy':59.0,'gpt1_race_middle_accuracy':62.9,'gpt1_sst2_accuracy':91.3,
|
| 530 |
+
'source_url':'https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf',
|
| 531 |
+
'note':'GPT-1 SST-2 91.3% is GLUE test, while this run reports public validation. GPT-1 RACE-M62.9% used combined RACE-middle/high training, while Wiggle used middle-only. Historical rows give context and are not a matched-model replication.'},
|
| 532 |
+
'tasks':tasks,'elapsed_seconds':sum(t['elapsed_seconds'] for t in tasks)}
|
| 533 |
+
(ROOT/'comparison.json').write_text(json.dumps(data,ensure_ascii=False,indent=2),encoding='utf-8')
|
| 534 |
+
|
| 535 |
+
def main():
|
| 536 |
+
parser=argparse.ArgumentParser(description=__doc__)
|
| 537 |
+
parser.add_argument('--task',choices=['all','sst2','race_middle'],default='all')
|
| 538 |
+
parser.add_argument('--resume',type=Path)
|
| 539 |
+
parser.add_argument('--verify-packed-loss-once',action='store_true',
|
| 540 |
+
help='Before the first RACE optimizer update, compare dense and packed auxiliary LM loss on one actual CUDA training candidate and record the result.')
|
| 541 |
+
args=parser.parse_args()
|
| 542 |
+
config=json.loads((ROOT/'train_config.json').read_text(encoding='utf-8'))
|
| 543 |
+
torch.set_num_threads(config['cpu_threads'])
|
| 544 |
+
try:
|
| 545 |
+
import psutil
|
| 546 |
+
psutil.Process().nice(psutil.BELOW_NORMAL_PRIORITY_CLASS)
|
| 547 |
+
except ImportError:
|
| 548 |
+
print('psutil unavailable; continuing with two CPU threads.',flush=True)
|
| 549 |
+
if not torch.cuda.is_available():
|
| 550 |
+
raise RuntimeError('CUDA required; CPU model execution is disabled.')
|
| 551 |
+
torch.cuda.set_device(0)
|
| 552 |
+
torch.set_float32_matmul_precision('high')
|
| 553 |
+
signal.signal(signal.SIGINT,stop_requested)
|
| 554 |
+
if args.resume:
|
| 555 |
+
resume_meta=torch.load(args.resume,map_location='cpu',weights_only=False,mmap=True)
|
| 556 |
+
selected=[resume_meta['task']]
|
| 557 |
+
config=resume_meta['config']
|
| 558 |
+
torch.set_num_threads(config['cpu_threads'])
|
| 559 |
+
del resume_meta
|
| 560 |
+
else:
|
| 561 |
+
selected=['sst2','race_middle'] if args.task=='all' else [args.task]
|
| 562 |
+
for task in selected:
|
| 563 |
+
if not run_task(task,config,args.resume,args.verify_packed_loss_once):
|
| 564 |
+
return
|
| 565 |
+
write_comparison()
|
| 566 |
+
print('All requested task adaptations complete.',flush=True)
|
| 567 |
+
|
| 568 |
+
if __name__=='__main__':
|
| 569 |
+
main()
|
task_finetuning/task_runtime.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Portable native WiggleGPT loader and exact GPT-2 tokenizer.
|
| 2 |
+
|
| 3 |
+
The model class is the byte-preserved source in native/model_bio.py.
|
| 4 |
+
Model construction and all forwards are CUDA only. Tokenization is CPU only.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
import base64
|
| 8 |
+
import hashlib
|
| 9 |
+
import os
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
import subprocess
|
| 12 |
+
import sys
|
| 13 |
+
ROOT = Path(__file__).resolve().parent
|
| 14 |
+
sys.path.insert(0, str(ROOT / 'native'))
|
| 15 |
+
|
| 16 |
+
DESKTOP_GPU_APPS = {
|
| 17 |
+
"shellhost.exe", "crossdeviceresume.exe", "lghub_system_tray.exe", "dcv2.exe",
|
| 18 |
+
"snippingtool.exe", "steelseriesggclient.exe", "explorer.exe", "textinputhost.exe",
|
| 19 |
+
"startmenuexperiencehost.exe", "searchhost.exe", "shellexperiencehost.exe",
|
| 20 |
+
"msedgewebview2.exe", "windowsterminal.exe", "chrome.exe", "msedge.exe", "firefox.exe",
|
| 21 |
+
"chatgpt.exe", "codex-computer-use-swift.exe", "razerappengine.exe", "notepad++.exe",
|
| 22 |
+
"obsidian.exe", "dwm.exe",
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
def get_encoder():
|
| 26 |
+
import tiktoken
|
| 27 |
+
from tiktoken_ext.openai_public import r50k_pat_str
|
| 28 |
+
|
| 29 |
+
raw = (ROOT / "tokenizer" / "gpt2.tiktoken").read_bytes()
|
| 30 |
+
# Exact canonical GPT-2/r50k merge table, independent of network/cache paths.
|
| 31 |
+
if hashlib.sha256(raw).hexdigest() != "306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930":
|
| 32 |
+
raise ValueError("Bundled GPT-2 merge table is damaged.")
|
| 33 |
+
ranks = {base64.b64decode(token): int(rank) for token, rank in
|
| 34 |
+
(line.split() for line in raw.splitlines() if line)}
|
| 35 |
+
return tiktoken.Encoding(name="gpt2", pat_str=r50k_pat_str, mergeable_ranks=ranks,
|
| 36 |
+
special_tokens={"<|endoftext|>": 50256}, explicit_n_vocab=50257)
|
| 37 |
+
|
| 38 |
+
def check_gpu():
|
| 39 |
+
result = subprocess.run(
|
| 40 |
+
["nvidia-smi", "--query-compute-apps=pid,process_name", "--format=csv,noheader,nounits"],
|
| 41 |
+
capture_output=True, text=True, check=True,
|
| 42 |
+
)
|
| 43 |
+
busy = []
|
| 44 |
+
for line in result.stdout.splitlines():
|
| 45 |
+
fields = line.split(",", 1)
|
| 46 |
+
# Current Windows WDDM reports ordinary graphics clients as C+G too.
|
| 47 |
+
# Desktop rendering is covered by the measured VRAM cap; other jobs wait.
|
| 48 |
+
desktop = len(fields) == 2 and Path(fields[1].strip()).name.lower() in DESKTOP_GPU_APPS
|
| 49 |
+
if fields[0].strip().isdigit() and int(fields[0]) != os.getpid() and not desktop:
|
| 50 |
+
busy.append(line.strip())
|
| 51 |
+
if busy:
|
| 52 |
+
raise RuntimeError("Another GPU job is running. Finish it before loading chat: " + "; ".join(busy))
|
| 53 |
+
|
| 54 |
+
def load_model(checkpoint_path):
|
| 55 |
+
check_gpu()
|
| 56 |
+
import torch
|
| 57 |
+
from model_bio import GPT, GPTConfig
|
| 58 |
+
|
| 59 |
+
if not torch.cuda.is_available():
|
| 60 |
+
raise RuntimeError("WiggleGPT needs CUDA here; CPU model execution is disabled.")
|
| 61 |
+
torch.set_num_threads(2)
|
| 62 |
+
torch.cuda.set_device(0)
|
| 63 |
+
free, total = torch.cuda.mem_get_info()
|
| 64 |
+
budget = min(total - int(2.5 * 1024**3), free - int(0.5 * 1024**3))
|
| 65 |
+
if budget < 2 * 1024**3:
|
| 66 |
+
raise RuntimeError("Less than 2 GiB is available after leaving desktop room.")
|
| 67 |
+
torch.cuda.set_per_process_memory_fraction(budget / total, 0)
|
| 68 |
+
torch.set_float32_matmul_precision("high")
|
| 69 |
+
checkpoint_path = Path(checkpoint_path)
|
| 70 |
+
checkpoint = torch.load(checkpoint_path, map_location="cpu", mmap=True, weights_only=False)
|
| 71 |
+
config = GPTConfig(**checkpoint["model_args"])
|
| 72 |
+
# Construct directly on CUDA. In particular, never run a CPU model forward.
|
| 73 |
+
with torch.device("cuda:0"):
|
| 74 |
+
model = GPT(config)
|
| 75 |
+
state = {key.removeprefix("_orig_mod."): value for key, value in checkpoint["model"].items()}
|
| 76 |
+
model.load_state_dict(state, strict=True)
|
| 77 |
+
model.eval()
|
| 78 |
+
metadata = {"checkpoint": str(checkpoint_path), "saved_step": checkpoint.get("iter_num", checkpoint.get("step")),
|
| 79 |
+
"parameters": sum(parameter.numel() for parameter in model.parameters()),
|
| 80 |
+
"model_args": checkpoint["model_args"], "torch": torch.__version__}
|
| 81 |
+
del checkpoint, state
|
| 82 |
+
encoder = get_encoder()
|
| 83 |
+
print(f"CUDA: {torch.cuda.get_device_name(0)}; {metadata['parameters']:,} parameters; "
|
| 84 |
+
f"{budget / 1024**3:.2f} GiB memory cap", flush=True)
|
| 85 |
+
print(f"Checkpoint: {checkpoint_path}", flush=True)
|
| 86 |
+
return model, config, encoder, metadata
|
task_finetuning/tokenizer/gpt2.tiktoken
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
task_finetuning/train_config.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"starting_checkpoint": "../checkpoints/pretrained/ckpt.pt",
|
| 3 |
+
"epochs": 3,
|
| 4 |
+
"effective_batch_size": 32,
|
| 5 |
+
"learning_rate": 6.25e-05,
|
| 6 |
+
"warmup_fraction": 0.002,
|
| 7 |
+
"schedule": "linear warmup then linear decay, matching GPT-1 task adaptation",
|
| 8 |
+
"lm_coefficient": 0.5,
|
| 9 |
+
"classifier_dropout": 0.1,
|
| 10 |
+
"weight_decay": 0.01,
|
| 11 |
+
"betas": [
|
| 12 |
+
0.9,
|
| 13 |
+
0.999
|
| 14 |
+
],
|
| 15 |
+
"epsilon": 1e-08,
|
| 16 |
+
"grad_clip": 1.0,
|
| 17 |
+
"max_length": 512,
|
| 18 |
+
"seed": 42,
|
| 19 |
+
"cpu_threads": 2,
|
| 20 |
+
"desktop_reserve_gib": 2.5,
|
| 21 |
+
"max_process_gib": 9.0,
|
| 22 |
+
"checkpoint_interval": 500,
|
| 23 |
+
"log_interval": 25,
|
| 24 |
+
"max_consecutive_nonfinite": 8,
|
| 25 |
+
"tasks": {
|
| 26 |
+
"sst2": {
|
| 27 |
+
"microbatch_size": 16,
|
| 28 |
+
"gradient_checkpointing": false,
|
| 29 |
+
"eval_interval": 1000
|
| 30 |
+
},
|
| 31 |
+
"race_middle": {
|
| 32 |
+
"microbatch_size": 2,
|
| 33 |
+
"gradient_checkpointing": true,
|
| 34 |
+
"eval_interval": 400
|
| 35 |
+
}
|
| 36 |
+
}
|
| 37 |
+
}
|
task_finetuning/train_tasks.py
ADDED
|
@@ -0,0 +1,451 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Full-model GPT-1-style task fine-tuning of original WiggleGPT.
|
| 2 |
+
|
| 3 |
+
Native model_bio.py is imported unchanged. A final-state classifier is added;
|
| 4 |
+
its language-model objective uses the native tied projection. No CPU forwards.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import argparse
|
| 9 |
+
from collections import Counter
|
| 10 |
+
from datetime import datetime
|
| 11 |
+
from zoneinfo import ZoneInfo
|
| 12 |
+
import gc
|
| 13 |
+
import hashlib
|
| 14 |
+
import json
|
| 15 |
+
import math
|
| 16 |
+
import os
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
import random
|
| 19 |
+
import shutil
|
| 20 |
+
import signal
|
| 21 |
+
import sys
|
| 22 |
+
import time
|
| 23 |
+
|
| 24 |
+
ROOT = Path(__file__).resolve().parent
|
| 25 |
+
NATIVE = ROOT / 'native'
|
| 26 |
+
sys.path.insert(0, str(NATIVE))
|
| 27 |
+
from task_runtime import load_model
|
| 28 |
+
import model_bio
|
| 29 |
+
import torch
|
| 30 |
+
from torch import nn
|
| 31 |
+
from torch.nn import functional as F
|
| 32 |
+
|
| 33 |
+
STOP = False
|
| 34 |
+
MARKERS = {'start': 50257, 'delimiter': 50258, 'classify': 50259, 'pad': 50256}
|
| 35 |
+
|
| 36 |
+
def stop_requested(signum, frame):
|
| 37 |
+
global STOP
|
| 38 |
+
STOP = True
|
| 39 |
+
print('Stop requested; finishing the update and saving latest.pt.', flush=True)
|
| 40 |
+
|
| 41 |
+
def digest(path):
|
| 42 |
+
with Path(path).open('rb') as stream:
|
| 43 |
+
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
| 44 |
+
|
| 45 |
+
class TaskModel(nn.Module):
|
| 46 |
+
def __init__(self, base, task, dropout):
|
| 47 |
+
super().__init__()
|
| 48 |
+
self.base = base
|
| 49 |
+
self.task = task
|
| 50 |
+
self.dropout = dropout
|
| 51 |
+
# Added linear classifier is the requested GPT-1 task head, not a backbone replacement.
|
| 52 |
+
with torch.device('cuda:0'):
|
| 53 |
+
self.classifier = nn.Linear(base.config.n_embd, 2 if task == 'sst2' else 1)
|
| 54 |
+
nn.init.normal_(self.classifier.weight, mean=0.0, std=0.02)
|
| 55 |
+
nn.init.zeros_(self.classifier.bias)
|
| 56 |
+
self.hidden = None
|
| 57 |
+
self.base.transformer.ln_f.register_forward_hook(self.capture_hidden)
|
| 58 |
+
|
| 59 |
+
def capture_hidden(self, module, args, output):
|
| 60 |
+
self.hidden = output
|
| 61 |
+
|
| 62 |
+
def forward(self, tokens, lengths, auxiliary=False):
|
| 63 |
+
batch, choices, width = tokens.shape
|
| 64 |
+
flat = tokens.reshape(batch * choices, width)
|
| 65 |
+
self.base(flat)
|
| 66 |
+
hidden = self.hidden
|
| 67 |
+
self.hidden = None
|
| 68 |
+
positions = lengths.reshape(-1) - 1
|
| 69 |
+
features = hidden[torch.arange(batch * choices, device=flat.device), positions]
|
| 70 |
+
features = features.reshape(batch, choices, -1)
|
| 71 |
+
# GPT-1's multiple-choice head shares a dropout feature mask across options.
|
| 72 |
+
if self.training and self.dropout:
|
| 73 |
+
mask = F.dropout(torch.ones((batch, 1, features.shape[-1]), device=features.device,
|
| 74 |
+
dtype=features.dtype), p=self.dropout, training=True)
|
| 75 |
+
features = features * mask
|
| 76 |
+
scores = self.classifier(features).squeeze(1) if self.task == 'sst2' else self.classifier(features).squeeze(-1)
|
| 77 |
+
lm_loss = None
|
| 78 |
+
if auxiliary:
|
| 79 |
+
logits = self.base.lm_head(hidden[:, :-1])
|
| 80 |
+
targets = flat[:, 1:].long()
|
| 81 |
+
per_token = F.cross_entropy(logits.float().reshape(-1, logits.shape[-1]),
|
| 82 |
+
targets.reshape(-1), reduction='none').view_as(targets)
|
| 83 |
+
mask = torch.arange(width - 1, device=flat.device)[None, :] < (lengths.reshape(-1, 1) - 1)
|
| 84 |
+
# Average within each candidate sequence, then across all candidates.
|
| 85 |
+
lm_loss = ((per_token * mask).sum(-1) / mask.sum(-1).clamp_min(1)).mean()
|
| 86 |
+
return scores.float(), lm_loss
|
| 87 |
+
|
| 88 |
+
def load_data(task, device):
|
| 89 |
+
prefix = 'sst2' if task == 'sst2' else 'race_middle'
|
| 90 |
+
splits = ['train', 'validation'] + (['test'] if task != 'sst2' else [])
|
| 91 |
+
data = {}
|
| 92 |
+
for split in splits:
|
| 93 |
+
path = ROOT / 'data' / f'{prefix}_{split}_encoded.pt'
|
| 94 |
+
if not path.exists():
|
| 95 |
+
# Preparation script may use the compact .pt suffix.
|
| 96 |
+
path = ROOT / 'data' / f'{prefix}_{split}.pt'
|
| 97 |
+
cache = torch.load(path, map_location='cpu', weights_only=False, mmap=True)
|
| 98 |
+
# Both conventional names are accepted; contents stay unchanged.
|
| 99 |
+
ids = cache.get('input_ids', cache.get('tokens'))
|
| 100 |
+
data[split] = {'input_ids': ids.to(device), 'lengths': cache['lengths'].to(device),
|
| 101 |
+
'labels': cache['labels'].to(device), 'cache_path': str(path),
|
| 102 |
+
'metadata': {k:v for k,v in cache.items() if not torch.is_tensor(v)}}
|
| 103 |
+
return data
|
| 104 |
+
|
| 105 |
+
def select_batch(data, index):
|
| 106 |
+
return data['input_ids'][index], data['lengths'][index], data['labels'][index]
|
| 107 |
+
|
| 108 |
+
def evaluate(model, data, batch_size, fixed_indices=None):
|
| 109 |
+
model.eval()
|
| 110 |
+
predictions, losses = [], []
|
| 111 |
+
if fixed_indices is None:
|
| 112 |
+
fixed_indices = torch.arange(data['labels'].shape[0], device='cuda:0')
|
| 113 |
+
# AMP-COMPLEX: native weights/activations are real; native RoPE explicitly casts its complex rotation to float32.
|
| 114 |
+
with torch.inference_mode(), torch.autocast('cuda', dtype=torch.bfloat16): # lint: ok AMP-COMPLEX real native weights; RoPE explicitly uses float32 complex rotation
|
| 115 |
+
for start in range(0, len(fixed_indices), batch_size):
|
| 116 |
+
indices = fixed_indices[start:start + batch_size]
|
| 117 |
+
x, lengths, labels = select_batch(data, indices)
|
| 118 |
+
logits, _ = model(x, lengths)
|
| 119 |
+
predictions.append(logits.argmax(-1))
|
| 120 |
+
losses.append(F.cross_entropy(logits, labels, reduction='sum'))
|
| 121 |
+
predicted = torch.cat(predictions)
|
| 122 |
+
labels = data['labels'][fixed_indices]
|
| 123 |
+
correct = int((predicted == labels).sum())
|
| 124 |
+
counts = torch.bincount(labels, minlength=2 if model.task == 'sst2' else 4).cpu().tolist()
|
| 125 |
+
return {'correct': correct, 'total': len(labels), 'accuracy': correct / len(labels),
|
| 126 |
+
'loss': float(torch.stack(losses).sum() / len(labels)),
|
| 127 |
+
'predictions': predicted.cpu().tolist(), 'gold_label_counts': counts,
|
| 128 |
+
'majority_accuracy': max(counts) / len(labels)}
|
| 129 |
+
|
| 130 |
+
def optimizer_for(model, config):
|
| 131 |
+
decay, no_decay = [], []
|
| 132 |
+
protected = ('omega', 'phi', 'baseline', 'norm', 'bias', 'scale', 'phase')
|
| 133 |
+
for name, parameter in model.named_parameters():
|
| 134 |
+
if parameter.ndim >= 2 and not parameter.is_complex() and not any(word in name for word in protected):
|
| 135 |
+
decay.append(parameter)
|
| 136 |
+
else:
|
| 137 |
+
no_decay.append(parameter)
|
| 138 |
+
print(f'Decay: {sum(p.numel() for p in decay):,}; protected/no decay: {sum(p.numel() for p in no_decay):,}', flush=True)
|
| 139 |
+
return torch.optim.AdamW([{'params':decay, 'weight_decay':config['weight_decay']},
|
| 140 |
+
{'params':no_decay, 'weight_decay':0.0}],
|
| 141 |
+
lr=config['learning_rate'], betas=tuple(config['betas']), eps=config['epsilon'], foreach=True)
|
| 142 |
+
|
| 143 |
+
def schedule(step, total, config):
|
| 144 |
+
warmup = max(1, round(total * config['warmup_fraction']))
|
| 145 |
+
if step < warmup:
|
| 146 |
+
return config['learning_rate'] * (step + 1) / warmup
|
| 147 |
+
return config['learning_rate'] * max(0.0, (total - step) / max(1, total - warmup))
|
| 148 |
+
|
| 149 |
+
def atomic_save(path, payload):
|
| 150 |
+
temporary = path.with_suffix('.pending.pt')
|
| 151 |
+
torch.save(payload, temporary)
|
| 152 |
+
temporary.replace(path)
|
| 153 |
+
|
| 154 |
+
def rng_state(generator):
|
| 155 |
+
return {'python': random.getstate(), 'torch': torch.get_rng_state(),
|
| 156 |
+
'cuda': torch.cuda.get_rng_state(0), 'batch_generator': generator.get_state()}
|
| 157 |
+
|
| 158 |
+
def restore_rng(state, generator):
|
| 159 |
+
random.setstate(state['python'])
|
| 160 |
+
torch.set_rng_state(state['torch'].cpu())
|
| 161 |
+
torch.cuda.set_rng_state(state['cuda'].cpu(), 0)
|
| 162 |
+
generator.set_state(state['batch_generator'].cpu())
|
| 163 |
+
|
| 164 |
+
def full_summary(result):
|
| 165 |
+
return {k:v for k,v in result.items() if k != 'predictions'}
|
| 166 |
+
|
| 167 |
+
def raw_rows(task, split):
|
| 168 |
+
path = ROOT / 'data' / f'{task}_{split}.jsonl'
|
| 169 |
+
return [json.loads(line) for line in path.read_text(encoding='utf-8').splitlines()]
|
| 170 |
+
|
| 171 |
+
def run_task(task, config, resume=None):
|
| 172 |
+
task_config = config['tasks'][task]
|
| 173 |
+
random.seed(config['seed'])
|
| 174 |
+
torch.manual_seed(config['seed'])
|
| 175 |
+
torch.cuda.manual_seed_all(config['seed'])
|
| 176 |
+
base, native_config, encoder, metadata = load_model((ROOT / config['starting_checkpoint']).resolve())
|
| 177 |
+
free, total = torch.cuda.mem_get_info()
|
| 178 |
+
cap = min(int(config['max_process_gib'] * 1024**3),
|
| 179 |
+
total - int(config['desktop_reserve_gib'] * 1024**3),
|
| 180 |
+
free + torch.cuda.memory_reserved() - int(0.5 * 1024**3))
|
| 181 |
+
torch.cuda.set_per_process_memory_fraction(cap / total, 0)
|
| 182 |
+
base.config.gradient_checkpointing = task_config['gradient_checkpointing']
|
| 183 |
+
metadata['checkpoint_sha256'] = digest((ROOT / config['starting_checkpoint']).resolve())
|
| 184 |
+
model = TaskModel(base, task, config['classifier_dropout'])
|
| 185 |
+
# Reuse three existing padding slots; real GPT-2 IDs and tokenizer are unchanged.
|
| 186 |
+
with torch.no_grad():
|
| 187 |
+
marker_ids = torch.tensor([MARKERS['start'],MARKERS['delimiter'],MARKERS['classify']], device='cuda:0')
|
| 188 |
+
base.transformer.wte.weight.index_copy_(0, marker_ids,
|
| 189 |
+
torch.randn((3, base.config.n_embd), device='cuda:0') * 0.02)
|
| 190 |
+
optimizer = optimizer_for(model, config)
|
| 191 |
+
data = load_data(task, 'cuda:0')
|
| 192 |
+
n_train = data['train']['labels'].shape[0]
|
| 193 |
+
per_epoch = math.ceil(n_train / config['effective_batch_size'])
|
| 194 |
+
total_steps = per_epoch * config['epochs']
|
| 195 |
+
generator = torch.Generator(device='cuda:0').manual_seed(config['seed'] + 123)
|
| 196 |
+
if resume:
|
| 197 |
+
checkpoint = torch.load(resume, map_location='cpu', weights_only=False, mmap=True)
|
| 198 |
+
model.load_state_dict(checkpoint['model'], strict=True)
|
| 199 |
+
optimizer.load_state_dict(checkpoint['optimizer'])
|
| 200 |
+
state = checkpoint['state']
|
| 201 |
+
order = checkpoint['order'].to('cuda:0') if checkpoint['order'] is not None else None
|
| 202 |
+
restore_rng(checkpoint['rng'], generator)
|
| 203 |
+
run_dir = Path(resume).resolve().parent
|
| 204 |
+
del checkpoint
|
| 205 |
+
else:
|
| 206 |
+
run_dir = ROOT / 'runs' / f'train_tasks_{datetime.now(ZoneInfo("Europe/London")).strftime("%Y-%m-%d_%H-%M-%S")}_{task}'
|
| 207 |
+
run_dir.mkdir(parents=True, exist_ok=False)
|
| 208 |
+
state = {'step':0, 'epoch':0, 'offset':0, 'best_accuracy':-1.0, 'best_loss':float('inf'),
|
| 209 |
+
'best_step':0, 'nonfinite_updates':0, 'consecutive_nonfinite':0, 'last_eval_step':-1,
|
| 210 |
+
'elapsed_seconds':0.0, 'lr_scale':1.0}
|
| 211 |
+
order = None
|
| 212 |
+
(run_dir / 'config.json').write_text(json.dumps(config, indent=2), encoding='utf-8')
|
| 213 |
+
shutil.copy2(__file__, run_dir / 'train_tasks.py')
|
| 214 |
+
shutil.copy2(model_bio.__file__, run_dir / 'model_bio.py')
|
| 215 |
+
provenance = {'model_source_sha256':digest(model_bio.__file__), 'trainer_sha256':digest(__file__),
|
| 216 |
+
'starting_checkpoint':metadata, 'torch':torch.__version__, 'markers':MARKERS,
|
| 217 |
+
'native_backbone_dropout':base.config.dropout,
|
| 218 |
+
'data':{split:d['metadata'] for split,d in data.items()},
|
| 219 |
+
'all_backbone_parameters_train':True}
|
| 220 |
+
(run_dir/'provenance.json').write_text(json.dumps(provenance, indent=2), encoding='utf-8')
|
| 221 |
+
log = (run_dir / (run_dir.name + '.log')).open('a', encoding='utf-8', buffering=1)
|
| 222 |
+
def emit(event):
|
| 223 |
+
event.update(task=task)
|
| 224 |
+
line = json.dumps(event, ensure_ascii=False)
|
| 225 |
+
print(line, flush=True)
|
| 226 |
+
log.write(line + '\n')
|
| 227 |
+
with (run_dir/'metrics.jsonl').open('a',encoding='utf-8') as stream:
|
| 228 |
+
stream.write(line + '\n')
|
| 229 |
+
def save(name):
|
| 230 |
+
atomic_save(run_dir/name, {'model':model.state_dict(), 'optimizer':optimizer.state_dict(),
|
| 231 |
+
'state':dict(state), 'order':order, 'rng':rng_state(generator), 'config':config,
|
| 232 |
+
'task':task, 'model_args':vars(base.config), 'markers':MARKERS,
|
| 233 |
+
'source_hashes':{'model':digest(model_bio.__file__),'trainer':digest(__file__)}})
|
| 234 |
+
evaluation_batch = 16 if task == 'sst2' else 4
|
| 235 |
+
heldout = 'validation' if task == 'sst2' else 'test'
|
| 236 |
+
before_path = run_dir/'before.json'
|
| 237 |
+
start_time = time.perf_counter()
|
| 238 |
+
elapsed_prior = state['elapsed_seconds']
|
| 239 |
+
emit({'event':'start','step':state['step'],'train_examples':n_train,'epochs':config['epochs'],
|
| 240 |
+
'total_steps':total_steps,'microbatch':task_config['microbatch_size'],
|
| 241 |
+
'effective_batch':config['effective_batch_size'],'memory_cap_gib':round(cap/1024**3,2),
|
| 242 |
+
'trainable_parameters':sum(p.numel() for p in model.parameters() if p.requires_grad),
|
| 243 |
+
'device':torch.cuda.get_device_name(0)})
|
| 244 |
+
if not before_path.exists():
|
| 245 |
+
before = evaluate(model, data[heldout], evaluation_batch)
|
| 246 |
+
before_path.write_text(json.dumps(before,indent=2),encoding='utf-8')
|
| 247 |
+
emit({'event':'before','heldout':heldout,**full_summary(before)})
|
| 248 |
+
else:
|
| 249 |
+
before = json.loads(before_path.read_text(encoding='utf-8'))
|
| 250 |
+
train_fixed = torch.arange(min(128,n_train), device='cuda:0')
|
| 251 |
+
def evaluate_and_save():
|
| 252 |
+
validation = evaluate(model, data['validation'], evaluation_batch)
|
| 253 |
+
train_eval = evaluate(model, data['train'], evaluation_batch, train_fixed)
|
| 254 |
+
changed = (validation['accuracy'] > state['best_accuracy'] or
|
| 255 |
+
(validation['accuracy'] == state['best_accuracy'] and validation['loss'] < state['best_loss']))
|
| 256 |
+
if changed:
|
| 257 |
+
state['best_accuracy'] = validation['accuracy']
|
| 258 |
+
state['best_loss'] = validation['loss']
|
| 259 |
+
state['best_step'] = state['step']
|
| 260 |
+
state['last_eval_step'] = state['step']
|
| 261 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter() - start_time
|
| 262 |
+
# Once-per-evaluation diagnostic over twelve oscillator tensors, not Python-built training batches.
|
| 263 |
+
omegas = torch.cat([p.detach().reshape(-1) for name,p in model.named_parameters() if name.endswith('.omega')]) # lint: ok STACK-COMPREHENSION evaluation statistics over twelve oscillator tensors
|
| 264 |
+
emit({'event':'eval','step':state['step'],'epoch':state['epoch'],'offset':state['offset'],
|
| 265 |
+
'train':full_summary(train_eval),'validation':full_summary(validation),
|
| 266 |
+
'best_step':state['best_step'],'omega_std':float(omegas.std(correction=0)),
|
| 267 |
+
'peak_allocated_gib':round(torch.cuda.max_memory_allocated()/1024**3,3),
|
| 268 |
+
'sample':{'gold':int(data['validation']['labels'][0]),'predicted':validation['predictions'][0]},
|
| 269 |
+
'elapsed_seconds':round(state['elapsed_seconds'],1)})
|
| 270 |
+
if changed:
|
| 271 |
+
save('best.pt')
|
| 272 |
+
save('latest.pt')
|
| 273 |
+
model.train()
|
| 274 |
+
if state['last_eval_step'] < 0:
|
| 275 |
+
evaluate_and_save()
|
| 276 |
+
model.train()
|
| 277 |
+
accumulated = torch.zeros(3,device='cuda:0')
|
| 278 |
+
logged_steps = 0
|
| 279 |
+
log_start = time.perf_counter()
|
| 280 |
+
stopping_cause = 'epochs_complete'
|
| 281 |
+
while state['epoch'] < config['epochs']:
|
| 282 |
+
if order is None:
|
| 283 |
+
order = torch.randperm(n_train,device='cuda:0',generator=generator)
|
| 284 |
+
while state['offset'] < n_train:
|
| 285 |
+
end = min(n_train,state['offset']+config['effective_batch_size'])
|
| 286 |
+
indices = order[state['offset']:end]
|
| 287 |
+
effective = len(indices)
|
| 288 |
+
lr = schedule(state['step'], total_steps, config) * state['lr_scale']
|
| 289 |
+
for group in optimizer.param_groups:
|
| 290 |
+
group['lr'] = lr
|
| 291 |
+
optimizer.zero_grad(set_to_none=True)
|
| 292 |
+
batch_values = torch.zeros(3,device='cuda:0')
|
| 293 |
+
for position in range(0,effective,task_config['microbatch_size']):
|
| 294 |
+
micro = indices[position:position+task_config['microbatch_size']]
|
| 295 |
+
x,lengths,labels = select_batch(data['train'],micro)
|
| 296 |
+
# AMP-COMPLEX: real-valued model; native rotary complex operations remain explicitly float32.
|
| 297 |
+
with torch.autocast('cuda',dtype=torch.bfloat16): # lint: ok AMP-COMPLEX real native weights; RoPE explicitly uses float32 complex rotation
|
| 298 |
+
logits,lm_loss = model(x,lengths,auxiliary=True)
|
| 299 |
+
classification_loss = F.cross_entropy(logits,labels)
|
| 300 |
+
loss = classification_loss + config['lm_coefficient'] * lm_loss
|
| 301 |
+
weighted = loss * (len(micro)/effective)
|
| 302 |
+
weighted.backward()
|
| 303 |
+
batch_values += torch.stack((loss.detach(),classification_loss.detach(),lm_loss.detach())) * (len(micro)/effective)
|
| 304 |
+
norm = torch.nn.utils.clip_grad_norm_(model.parameters(),config['grad_clip'])
|
| 305 |
+
if torch.isfinite(norm):
|
| 306 |
+
optimizer.step()
|
| 307 |
+
state['consecutive_nonfinite'] = 0
|
| 308 |
+
else:
|
| 309 |
+
state['nonfinite_updates'] += 1
|
| 310 |
+
state['consecutive_nonfinite'] += 1
|
| 311 |
+
if state['consecutive_nonfinite'] >= 2:
|
| 312 |
+
state['lr_scale'] *= 0.5
|
| 313 |
+
emit({'event':'nonfinite_update_skipped','step':state['step']+1,
|
| 314 |
+
'count':state['nonfinite_updates'],'lr_scale':state['lr_scale']})
|
| 315 |
+
state['step'] += 1
|
| 316 |
+
state['offset'] = end
|
| 317 |
+
accumulated += batch_values
|
| 318 |
+
logged_steps += 1
|
| 319 |
+
if state['step'] == 1 or state['step'] % config['log_interval'] == 0:
|
| 320 |
+
# One host transfer per logging interval; hot microbatches remain on CUDA.
|
| 321 |
+
values = (accumulated/logged_steps).cpu().tolist() # lint: ok SYNC-IN-LOOP once per logging interval
|
| 322 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter()-start_time
|
| 323 |
+
emit({'event':'train','step':state['step'],'total_steps':total_steps,
|
| 324 |
+
'epoch_fraction':round(state['epoch']+end/n_train,4),'loss':values[0],
|
| 325 |
+
'classification_loss':values[1],'lm_loss':values[2],'lr':lr,
|
| 326 |
+
'updates_per_second':round(logged_steps/(time.perf_counter()-log_start),3),
|
| 327 |
+
'peak_allocated_gib':round(torch.cuda.max_memory_allocated()/1024**3,3),
|
| 328 |
+
'nonfinite_updates':state['nonfinite_updates'],'elapsed_seconds':round(state['elapsed_seconds'],1)})
|
| 329 |
+
accumulated.zero_(); logged_steps=0;log_start=time.perf_counter()
|
| 330 |
+
if state['step'] % task_config['eval_interval'] == 0:
|
| 331 |
+
evaluate_and_save()
|
| 332 |
+
log_start=time.perf_counter()
|
| 333 |
+
elif state['step'] % config['checkpoint_interval'] == 0:
|
| 334 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter()-start_time
|
| 335 |
+
save('latest.pt')
|
| 336 |
+
if STOP or state['consecutive_nonfinite'] >= config['max_consecutive_nonfinite']:
|
| 337 |
+
stopping_cause = 'user_interrupt' if STOP else 'nonfinite_limit'
|
| 338 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter()-start_time
|
| 339 |
+
save('latest.pt')
|
| 340 |
+
emit({'event':'stopped','stopping_cause':stopping_cause,'step':state['step']})
|
| 341 |
+
log.close()
|
| 342 |
+
return False
|
| 343 |
+
state['epoch'] += 1
|
| 344 |
+
state['offset'] = 0
|
| 345 |
+
order = None
|
| 346 |
+
if state['last_eval_step'] != state['step']:
|
| 347 |
+
evaluate_and_save()
|
| 348 |
+
else:
|
| 349 |
+
save('latest.pt')
|
| 350 |
+
# Final held-out scores use the best validation-selected checkpoint.
|
| 351 |
+
best = torch.load(run_dir/'best.pt',map_location='cpu',weights_only=False,mmap=True)
|
| 352 |
+
model.load_state_dict(best['model'],strict=True)
|
| 353 |
+
best_step = best['state']['step']
|
| 354 |
+
del best
|
| 355 |
+
after = evaluate(model,data[heldout],evaluation_batch)
|
| 356 |
+
(run_dir/'after.json').write_text(json.dumps(after,indent=2),encoding='utf-8')
|
| 357 |
+
rows = raw_rows(task,heldout)
|
| 358 |
+
examples = []
|
| 359 |
+
for row_index,(row,pb,pa) in enumerate(zip(rows,before['predictions'],after['predictions'])):
|
| 360 |
+
example = {'id':f'{task}/{heldout}/{row_index}','input':row.get('text',row.get('article','')),
|
| 361 |
+
'prompt':row.get('question',''),'options':row.get('options',['negative','positive']),
|
| 362 |
+
'gold':row['label'],'predicted_before':pb,'predicted_after':pa,
|
| 363 |
+
'correct_before':pb==row['label'],'correct_after':pa==row['label']}
|
| 364 |
+
examples.append(example)
|
| 365 |
+
(run_dir/'heldout_predictions.json').write_text(json.dumps(examples,ensure_ascii=False,indent=2),encoding='utf-8')
|
| 366 |
+
probe_count = 100 if task=='sst2' else 40
|
| 367 |
+
probe = {'before_correct':sum(e['correct_before'] for e in examples[:probe_count]),
|
| 368 |
+
'after_correct':sum(e['correct_after'] for e in examples[:probe_count]),'total':probe_count}
|
| 369 |
+
state['elapsed_seconds'] = elapsed_prior+time.perf_counter()-start_time
|
| 370 |
+
summary = {'id':task,'name':'SST-2 sentiment' if task=='sst2' else 'RACE middle-school reading',
|
| 371 |
+
'train_examples':n_train,'validation_examples':len(data['validation']['labels']),
|
| 372 |
+
'test_examples':len(data['test']['labels']) if task!='sst2' else 1821,
|
| 373 |
+
'heldout_split':heldout,'baseline':{'label':'Before task training (same classifier)',**full_summary(before)},
|
| 374 |
+
'finetuned':full_summary(after),'chance_accuracy':0.5 if task=='sst2' else 0.25,
|
| 375 |
+
'majority_accuracy':after['majority_accuracy'],'paired_earlier_probe':probe,
|
| 376 |
+
'training':{'epochs':config['epochs'],'max_steps':total_steps,'steps':state['step'],
|
| 377 |
+
'batch_size':config['effective_batch_size'],'microbatch_size':task_config['microbatch_size'],
|
| 378 |
+
'gradient_accumulation':config['effective_batch_size']//task_config['microbatch_size'],
|
| 379 |
+
'learning_rate':config['learning_rate'],'max_length':config['max_length'],
|
| 380 |
+
'lm_coefficient':config['lm_coefficient'],'stopping_cause':stopping_cause,
|
| 381 |
+
'classifier_dropout':config['classifier_dropout'],'native_backbone_dropout':base.config.dropout,
|
| 382 |
+
'nonfinite_updates':state['nonfinite_updates'],'best_step':best_step,
|
| 383 |
+
'peak_allocated_gib':round(torch.cuda.max_memory_allocated()/1024**3,3),
|
| 384 |
+
'trainable_parameters':sum(p.numel() for p in model.parameters()),
|
| 385 |
+
'head':'768-to-2 linear' if task=='sst2' else 'shared 768-to-1 linear over four answers',
|
| 386 |
+
'checkpoint_selection':'highest validation accuracy, lower validation classification loss breaks ties',
|
| 387 |
+
'details':'Full native backbone trained. Zero decay on oscillator and 1-D parameters. Linear GPT-1 warmup/decay, AdamW, gradient clip1.0. Original backbone dropout retained; classifier dropout0.1. Three unused padding-vocabulary slots initialized for task markers.'},
|
| 388 |
+
'examples':examples,'checkpoint':{'name':str((run_dir/'best.pt').relative_to(ROOT)),
|
| 389 |
+
'sha256':digest(run_dir/'best.pt')},
|
| 390 |
+
'elapsed_seconds':state['elapsed_seconds']}
|
| 391 |
+
(run_dir/'summary.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2),encoding='utf-8')
|
| 392 |
+
emit({'event':'complete','step':state['step'],'best_step':best_step,'stopping_cause':stopping_cause,
|
| 393 |
+
'heldout':heldout,'before':full_summary(before),'after':full_summary(after),
|
| 394 |
+
'paired_probe':probe,'elapsed_seconds':round(state['elapsed_seconds'],1)})
|
| 395 |
+
log.close()
|
| 396 |
+
del model,base,optimizer,data
|
| 397 |
+
gc.collect();torch.cuda.empty_cache()
|
| 398 |
+
return True
|
| 399 |
+
|
| 400 |
+
def write_comparison():
|
| 401 |
+
tasks=[]
|
| 402 |
+
for task in ['sst2','race_middle']:
|
| 403 |
+
paths=sorted((ROOT/'runs').glob(f'*_{task}/summary.json'),key=lambda path:path.stat().st_mtime)
|
| 404 |
+
if not paths:
|
| 405 |
+
return
|
| 406 |
+
tasks.append(json.loads(paths[-1].read_text(encoding='utf-8')))
|
| 407 |
+
data={'date':datetime.now(ZoneInfo('Europe/London')).isoformat(),
|
| 408 |
+
'scope':'Full SST-2 training and validation; full RACE-middle training, validation selection and held-out test. Historical GPT-1 figures use different training coverage and test sets.',
|
| 409 |
+
'method':'Two independent full-model adaptations from original pretrained WiggleGPT. Classifier cross-entropy plus0.5 times token-average auxiliary language-model cross-entropy, with GPT-1-style input markers and linear task heads.',
|
| 410 |
+
'control_method':'Identical task inputs and initialized classifier evaluated before task training; after scores use validation-selected trained checkpoint. Earlier language-model likelihood probes are separately identified.',
|
| 411 |
+
'historical':{'gpt1_race_accuracy':59.0,'gpt1_race_middle_accuracy':62.9,'gpt1_sst2_accuracy':91.3,
|
| 412 |
+
'source_url':'https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf',
|
| 413 |
+
'note':'GPT-1 SST-2 91.3% is GLUE test, while this run reports public validation. GPT-1 RACE-M62.9% used combined RACE-middle/high training, while Wiggle used middle-only. Historical rows give context and are not a matched-model replication.'},
|
| 414 |
+
'tasks':tasks,'elapsed_seconds':sum(t['elapsed_seconds'] for t in tasks)}
|
| 415 |
+
(ROOT/'comparison.json').write_text(json.dumps(data,ensure_ascii=False,indent=2),encoding='utf-8')
|
| 416 |
+
|
| 417 |
+
def main():
|
| 418 |
+
parser=argparse.ArgumentParser(description=__doc__)
|
| 419 |
+
parser.add_argument('--task',choices=['all','sst2','race_middle'],default='all')
|
| 420 |
+
parser.add_argument('--resume',type=Path)
|
| 421 |
+
parser.add_argument('--config',type=Path,default=ROOT/'train_config.json')
|
| 422 |
+
parser.add_argument('--starting-checkpoint',type=Path,help='Original pretrained checkpoint; overrides config for a new run.')
|
| 423 |
+
args=parser.parse_args()
|
| 424 |
+
config=json.loads(args.config.read_text(encoding='utf-8'))
|
| 425 |
+
if args.starting_checkpoint:
|
| 426 |
+
config['starting_checkpoint']=str(args.starting_checkpoint.expanduser().resolve())
|
| 427 |
+
torch.set_num_threads(config['cpu_threads'])
|
| 428 |
+
try:
|
| 429 |
+
import psutil
|
| 430 |
+
psutil.Process().nice(psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == 'nt' else 10)
|
| 431 |
+
except (ImportError, OSError):
|
| 432 |
+
print('Optional process priority setting unavailable; continuing with configured CPU threads.',flush=True)
|
| 433 |
+
if not torch.cuda.is_available():
|
| 434 |
+
raise RuntimeError('CUDA required; CPU model execution is disabled.')
|
| 435 |
+
torch.cuda.set_device(0)
|
| 436 |
+
torch.set_float32_matmul_precision('high')
|
| 437 |
+
signal.signal(signal.SIGINT,stop_requested)
|
| 438 |
+
if args.resume:
|
| 439 |
+
resume_meta=torch.load(args.resume,map_location='cpu',weights_only=False,mmap=True)
|
| 440 |
+
selected=[resume_meta['task']]
|
| 441 |
+
del resume_meta
|
| 442 |
+
else:
|
| 443 |
+
selected=['sst2','race_middle'] if args.task=='all' else [args.task]
|
| 444 |
+
for task in selected:
|
| 445 |
+
if not run_task(task,config,args.resume):
|
| 446 |
+
return
|
| 447 |
+
write_comparison()
|
| 448 |
+
print('All requested task adaptations complete.',flush=True)
|
| 449 |
+
|
| 450 |
+
if __name__=='__main__':
|
| 451 |
+
main()
|
task_finetuning/train_tasks_v3.py
ADDED
|
@@ -0,0 +1,571 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Full-model GPT-1-style task fine-tuning of original WiggleGPT, version 3.
|
| 2 |
+
|
| 3 |
+
Native model_bio.py is imported unchanged. A final-state classifier is added;
|
| 4 |
+
its language-model objective uses the native tied projection. No CPU forwards.
|
| 5 |
+
CPU length plans remove trailing padding; auxiliary LM projection skips padding.
|
| 6 |
+
RACE uses native gradient checkpointing only above 384 cropped tokens.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import argparse
|
| 11 |
+
from collections import Counter
|
| 12 |
+
from datetime import datetime
|
| 13 |
+
from zoneinfo import ZoneInfo
|
| 14 |
+
import gc
|
| 15 |
+
import hashlib
|
| 16 |
+
import json
|
| 17 |
+
import math
|
| 18 |
+
import os
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
import random
|
| 21 |
+
import shutil
|
| 22 |
+
import signal
|
| 23 |
+
import sys
|
| 24 |
+
import time
|
| 25 |
+
|
| 26 |
+
ROOT = Path(__file__).resolve().parent
|
| 27 |
+
NATIVE = ROOT / 'native'
|
| 28 |
+
sys.path.insert(0, str(NATIVE))
|
| 29 |
+
from task_runtime import load_model
|
| 30 |
+
import model_bio
|
| 31 |
+
import torch
|
| 32 |
+
from torch import nn
|
| 33 |
+
from torch.nn import functional as F
|
| 34 |
+
|
| 35 |
+
STOP = False
|
| 36 |
+
MARKERS = {'start': 50257, 'delimiter': 50258, 'classify': 50259, 'pad': 50256}
|
| 37 |
+
CHECKPOINT_THRESHOLD = 384
|
| 38 |
+
OPTIMIZATIONS = {
|
| 39 |
+
'version': 3,
|
| 40 |
+
'training_padding': 'CPU epoch plan crops each unchanged microbatch to its longest candidate, rounded up to 16 tokens (minimum 16).',
|
| 41 |
+
'auxiliary_lm': 'Project only valid next-token positions; weight each token by the inverse length of its own candidate, then average all candidates.',
|
| 42 |
+
'device_synchronization': 'One order copy to CPU per epoch; CPU-known output_size avoids synchronization when building packed CUDA indices.',
|
| 43 |
+
'question_order_and_effective_batch': 'Original seeded CUDA randperm, effective batch and microbatch membership retained.',
|
| 44 |
+
'native_backbone': 'Unchanged native model_bio.py; RACE checkpointing enabled when cropped width exceeds 384 and disabled at or below 384. SST-2 keeps its configured setting.',
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
def stop_requested(signum, frame):
|
| 48 |
+
global STOP
|
| 49 |
+
STOP = True
|
| 50 |
+
print('Stop requested; finishing the update and saving latest.pt.', flush=True)
|
| 51 |
+
|
| 52 |
+
def digest(path):
|
| 53 |
+
with Path(path).open('rb') as stream:
|
| 54 |
+
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
| 55 |
+
|
| 56 |
+
class TaskModel(nn.Module):
|
| 57 |
+
def __init__(self, base, task, dropout):
|
| 58 |
+
super().__init__()
|
| 59 |
+
self.base = base
|
| 60 |
+
self.task = task
|
| 61 |
+
self.dropout = dropout
|
| 62 |
+
# Added linear classifier is the requested GPT-1 task head, not a backbone replacement.
|
| 63 |
+
with torch.device('cuda:0'):
|
| 64 |
+
self.classifier = nn.Linear(base.config.n_embd, 2 if task == 'sst2' else 1)
|
| 65 |
+
nn.init.normal_(self.classifier.weight, mean=0.0, std=0.02)
|
| 66 |
+
nn.init.zeros_(self.classifier.bias)
|
| 67 |
+
self.hidden = None
|
| 68 |
+
self.last_loss_verification = None
|
| 69 |
+
self.base.transformer.ln_f.register_forward_hook(self.capture_hidden)
|
| 70 |
+
|
| 71 |
+
def capture_hidden(self, module, args, output):
|
| 72 |
+
self.hidden = output
|
| 73 |
+
|
| 74 |
+
def forward(self, tokens, lengths, auxiliary=False, packed_lm_tokens=None,
|
| 75 |
+
verify_packed=False):
|
| 76 |
+
batch, choices, width = tokens.shape
|
| 77 |
+
flat = tokens.reshape(batch * choices, width)
|
| 78 |
+
self.base(flat)
|
| 79 |
+
hidden = self.hidden
|
| 80 |
+
self.hidden = None
|
| 81 |
+
positions = lengths.reshape(-1) - 1
|
| 82 |
+
features = hidden[torch.arange(batch * choices, device=flat.device), positions]
|
| 83 |
+
features = features.reshape(batch, choices, -1)
|
| 84 |
+
# GPT-1's multiple-choice head shares a dropout feature mask across options.
|
| 85 |
+
if self.training and self.dropout:
|
| 86 |
+
mask = F.dropout(torch.ones((batch, 1, features.shape[-1]), device=features.device,
|
| 87 |
+
dtype=features.dtype), p=self.dropout, training=True)
|
| 88 |
+
features = features * mask
|
| 89 |
+
scores = self.classifier(features).squeeze(1) if self.task == 'sst2' else self.classifier(features).squeeze(-1)
|
| 90 |
+
lm_loss = None
|
| 91 |
+
if auxiliary:
|
| 92 |
+
# Average within each candidate sequence, then across all candidates.
|
| 93 |
+
# output_size is supplied by the CPU length plan, avoiding a CUDA
|
| 94 |
+
# synchronization to discover the size of a boolean-index result.
|
| 95 |
+
if packed_lm_tokens is None:
|
| 96 |
+
raise ValueError('Auxiliary LM requires the CPU-planned valid-token count.')
|
| 97 |
+
counts = lengths.reshape(-1) - 1
|
| 98 |
+
sequence_ids = torch.repeat_interleave(
|
| 99 |
+
torch.arange(batch * choices, device=flat.device), counts,
|
| 100 |
+
output_size=packed_lm_tokens)
|
| 101 |
+
starts = counts.cumsum(0) - counts
|
| 102 |
+
token_positions = (torch.arange(packed_lm_tokens, device=flat.device)
|
| 103 |
+
- starts[sequence_ids])
|
| 104 |
+
flat_positions = sequence_ids * width + token_positions
|
| 105 |
+
lm_inputs = hidden.reshape(-1, hidden.shape[-1]).index_select(0, flat_positions)
|
| 106 |
+
targets = flat.reshape(-1).index_select(0, flat_positions + 1).long()
|
| 107 |
+
logits = self.base.lm_head(lm_inputs)
|
| 108 |
+
per_token = F.cross_entropy(logits.float(), targets, reduction='none')
|
| 109 |
+
lm_loss = (per_token / counts[sequence_ids]).sum() / (batch * choices)
|
| 110 |
+
if verify_packed:
|
| 111 |
+
# Both paths use these same actual CUDA hidden states, tied
|
| 112 |
+
# projection, autocast dtype and per-candidate normalization.
|
| 113 |
+
# Limit the comparison to one real candidate to keep the
|
| 114 |
+
# optional once-only verification below the desktop VRAM cap.
|
| 115 |
+
with torch.no_grad():
|
| 116 |
+
dense_logits = self.base.lm_head(hidden[:1, :-1].detach())
|
| 117 |
+
dense_targets = flat[:1, 1:].long()
|
| 118 |
+
dense_tokens = F.cross_entropy(
|
| 119 |
+
dense_logits.float().reshape(-1, dense_logits.shape[-1]),
|
| 120 |
+
dense_targets.reshape(-1), reduction='none').view_as(dense_targets)
|
| 121 |
+
dense_mask = torch.arange(width - 1, device=flat.device)[None, :] < counts[:1, None]
|
| 122 |
+
dense_loss = ((dense_tokens * dense_mask).sum(-1)
|
| 123 |
+
/ counts[:1].clamp_min(1)).mean()
|
| 124 |
+
packed_first = ((per_token.detach() / counts[sequence_ids])
|
| 125 |
+
* (sequence_ids == 0)).sum()
|
| 126 |
+
values = torch.stack((dense_loss, packed_first,
|
| 127 |
+
(dense_loss - packed_first).abs())).cpu().tolist() # lint: ok COMPLEX-ABS real scalar CE losses
|
| 128 |
+
self.last_loss_verification = {
|
| 129 |
+
'passed': math.isclose(values[0], values[1], rel_tol=1e-4, abs_tol=1e-5),
|
| 130 |
+
'dense_loss': values[0], 'packed_loss': values[1],
|
| 131 |
+
'absolute_difference': values[2], 'relative_tolerance': 1e-4,
|
| 132 |
+
'absolute_tolerance': 1e-5, 'candidates_checked': 1,
|
| 133 |
+
'actual_training_candidates': batch * choices,
|
| 134 |
+
'packed_lm_tokens': packed_lm_tokens, 'cropped_width': width,
|
| 135 |
+
'projection_dtype': str(logits.dtype),
|
| 136 |
+
'device': str(hidden.device),
|
| 137 |
+
'comparison': 'Same actual training hidden states; dense masked CE versus packed valid-position CE, before the first optimizer update.',
|
| 138 |
+
}
|
| 139 |
+
return scores.float(), lm_loss
|
| 140 |
+
|
| 141 |
+
def load_data(task, device):
|
| 142 |
+
prefix = 'sst2' if task == 'sst2' else 'race_middle'
|
| 143 |
+
splits = ['train', 'validation'] + (['test'] if task != 'sst2' else [])
|
| 144 |
+
data = {}
|
| 145 |
+
for split in splits:
|
| 146 |
+
path = ROOT / 'data' / f'{prefix}_{split}_encoded.pt'
|
| 147 |
+
if not path.exists():
|
| 148 |
+
# Preparation script may use the compact .pt suffix.
|
| 149 |
+
path = ROOT / 'data' / f'{prefix}_{split}.pt'
|
| 150 |
+
cache = torch.load(path, map_location='cpu', weights_only=False, mmap=True)
|
| 151 |
+
# Both conventional names are accepted; contents stay unchanged.
|
| 152 |
+
ids = cache.get('input_ids', cache.get('tokens'))
|
| 153 |
+
data[split] = {'input_ids': ids.to(device), 'lengths': cache['lengths'].to(device),
|
| 154 |
+
'lengths_cpu': cache['lengths'].clone(),
|
| 155 |
+
'labels': cache['labels'].to(device), 'cache_path': str(path),
|
| 156 |
+
'metadata': {k:v for k,v in cache.items() if not torch.is_tensor(v)}}
|
| 157 |
+
return data
|
| 158 |
+
|
| 159 |
+
def select_batch(data, index, width=None):
|
| 160 |
+
tokens = data['input_ids'][index]
|
| 161 |
+
if width is not None:
|
| 162 |
+
tokens = tokens[..., :width]
|
| 163 |
+
return tokens, data['lengths'][index], data['labels'][index]
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def plan_epoch_microbatches(data, order, effective_batch_size, microbatch_size):
|
| 167 |
+
"""Plan CPU scalar lengths once, preserving the saved CUDA question order."""
|
| 168 |
+
ordered_lengths = data['lengths_cpu'].index_select(0, order.detach().cpu())
|
| 169 |
+
cached_width = data['input_ids'].shape[-1]
|
| 170 |
+
plan = {}
|
| 171 |
+
for start in range(0, len(order), effective_batch_size):
|
| 172 |
+
end = min(len(order), start + effective_batch_size)
|
| 173 |
+
for position in range(start, end, microbatch_size):
|
| 174 |
+
lengths = ordered_lengths[position:min(end, position + microbatch_size)]
|
| 175 |
+
width = min(cached_width, max(16, math.ceil(int(lengths.max()) / 16) * 16))
|
| 176 |
+
plan[position] = (width, int((lengths - 1).sum()))
|
| 177 |
+
return plan
|
| 178 |
+
|
| 179 |
+
def evaluate(model, data, batch_size, fixed_indices=None):
|
| 180 |
+
model.eval()
|
| 181 |
+
predictions, losses = [], []
|
| 182 |
+
if fixed_indices is None:
|
| 183 |
+
fixed_indices = torch.arange(data['labels'].shape[0], device='cuda:0')
|
| 184 |
+
# AMP-COMPLEX: native weights/activations are real; native RoPE explicitly casts its complex rotation to float32.
|
| 185 |
+
with torch.inference_mode(), torch.autocast('cuda', dtype=torch.bfloat16): # lint: ok AMP-COMPLEX real native weights; RoPE explicitly uses float32 complex rotation
|
| 186 |
+
for start in range(0, len(fixed_indices), batch_size):
|
| 187 |
+
indices = fixed_indices[start:start + batch_size]
|
| 188 |
+
x, lengths, labels = select_batch(data, indices)
|
| 189 |
+
logits, _ = model(x, lengths)
|
| 190 |
+
predictions.append(logits.argmax(-1))
|
| 191 |
+
losses.append(F.cross_entropy(logits, labels, reduction='sum'))
|
| 192 |
+
predicted = torch.cat(predictions)
|
| 193 |
+
labels = data['labels'][fixed_indices]
|
| 194 |
+
correct = int((predicted == labels).sum())
|
| 195 |
+
counts = torch.bincount(labels, minlength=2 if model.task == 'sst2' else 4).cpu().tolist()
|
| 196 |
+
return {'correct': correct, 'total': len(labels), 'accuracy': correct / len(labels),
|
| 197 |
+
'loss': float(torch.stack(losses).sum() / len(labels)),
|
| 198 |
+
'predictions': predicted.cpu().tolist(), 'gold_label_counts': counts,
|
| 199 |
+
'majority_accuracy': max(counts) / len(labels)}
|
| 200 |
+
|
| 201 |
+
def optimizer_for(model, config):
|
| 202 |
+
decay, no_decay = [], []
|
| 203 |
+
protected = ('omega', 'phi', 'baseline', 'norm', 'bias', 'scale', 'phase')
|
| 204 |
+
for name, parameter in model.named_parameters():
|
| 205 |
+
if parameter.ndim >= 2 and not parameter.is_complex() and not any(word in name for word in protected):
|
| 206 |
+
decay.append(parameter)
|
| 207 |
+
else:
|
| 208 |
+
no_decay.append(parameter)
|
| 209 |
+
print(f'Decay: {sum(p.numel() for p in decay):,}; protected/no decay: {sum(p.numel() for p in no_decay):,}', flush=True)
|
| 210 |
+
return torch.optim.AdamW([{'params':decay, 'weight_decay':config['weight_decay']},
|
| 211 |
+
{'params':no_decay, 'weight_decay':0.0}],
|
| 212 |
+
lr=config['learning_rate'], betas=tuple(config['betas']), eps=config['epsilon'], foreach=True)
|
| 213 |
+
|
| 214 |
+
def schedule(step, total, config):
|
| 215 |
+
warmup = max(1, round(total * config['warmup_fraction']))
|
| 216 |
+
if step < warmup:
|
| 217 |
+
return config['learning_rate'] * (step + 1) / warmup
|
| 218 |
+
return config['learning_rate'] * max(0.0, (total - step) / max(1, total - warmup))
|
| 219 |
+
|
| 220 |
+
def atomic_save(path, payload):
|
| 221 |
+
temporary = path.with_suffix('.pending.pt')
|
| 222 |
+
torch.save(payload, temporary)
|
| 223 |
+
temporary.replace(path)
|
| 224 |
+
|
| 225 |
+
def rng_state(generator):
|
| 226 |
+
return {'python': random.getstate(), 'torch': torch.get_rng_state(),
|
| 227 |
+
'cuda': torch.cuda.get_rng_state(0), 'batch_generator': generator.get_state()}
|
| 228 |
+
|
| 229 |
+
def restore_rng(state, generator):
|
| 230 |
+
random.setstate(state['python'])
|
| 231 |
+
torch.set_rng_state(state['torch'].cpu())
|
| 232 |
+
torch.cuda.set_rng_state(state['cuda'].cpu(), 0)
|
| 233 |
+
generator.set_state(state['batch_generator'].cpu())
|
| 234 |
+
|
| 235 |
+
def full_summary(result):
|
| 236 |
+
return {k:v for k,v in result.items() if k != 'predictions'}
|
| 237 |
+
|
| 238 |
+
def raw_rows(task, split):
|
| 239 |
+
path = ROOT / 'data' / f'{task}_{split}.jsonl'
|
| 240 |
+
return [json.loads(line) for line in path.read_text(encoding='utf-8').splitlines()]
|
| 241 |
+
|
| 242 |
+
def run_task(task, config, resume=None, verify_packed_loss_once=False):
|
| 243 |
+
task_config = config['tasks'][task]
|
| 244 |
+
random.seed(config['seed'])
|
| 245 |
+
torch.manual_seed(config['seed'])
|
| 246 |
+
torch.cuda.manual_seed_all(config['seed'])
|
| 247 |
+
base, native_config, encoder, metadata = load_model((ROOT / config['starting_checkpoint']).resolve())
|
| 248 |
+
free, total = torch.cuda.mem_get_info()
|
| 249 |
+
cap = min(int(config['max_process_gib'] * 1024**3),
|
| 250 |
+
total - int(config['desktop_reserve_gib'] * 1024**3),
|
| 251 |
+
free + torch.cuda.memory_reserved() - int(0.5 * 1024**3))
|
| 252 |
+
torch.cuda.set_per_process_memory_fraction(cap / total, 0)
|
| 253 |
+
base.config.gradient_checkpointing = task_config['gradient_checkpointing']
|
| 254 |
+
metadata['checkpoint_sha256'] = digest((ROOT / config['starting_checkpoint']).resolve())
|
| 255 |
+
model = TaskModel(base, task, config['classifier_dropout'])
|
| 256 |
+
# Reuse three existing padding slots; real GPT-2 IDs and tokenizer are unchanged.
|
| 257 |
+
with torch.no_grad():
|
| 258 |
+
marker_ids = torch.tensor([MARKERS['start'],MARKERS['delimiter'],MARKERS['classify']], device='cuda:0')
|
| 259 |
+
base.transformer.wte.weight.index_copy_(0, marker_ids,
|
| 260 |
+
torch.randn((3, base.config.n_embd), device='cuda:0') * 0.02)
|
| 261 |
+
optimizer = optimizer_for(model, config)
|
| 262 |
+
data = load_data(task, 'cuda:0')
|
| 263 |
+
n_train = data['train']['labels'].shape[0]
|
| 264 |
+
per_epoch = math.ceil(n_train / config['effective_batch_size'])
|
| 265 |
+
total_steps = per_epoch * config['epochs']
|
| 266 |
+
generator = torch.Generator(device='cuda:0').manual_seed(config['seed'] + 123)
|
| 267 |
+
if resume:
|
| 268 |
+
checkpoint = torch.load(resume, map_location='cpu', weights_only=False, mmap=True)
|
| 269 |
+
model.load_state_dict(checkpoint['model'], strict=True)
|
| 270 |
+
optimizer.load_state_dict(checkpoint['optimizer'])
|
| 271 |
+
state = checkpoint['state']
|
| 272 |
+
order = checkpoint['order'].to('cuda:0') if checkpoint['order'] is not None else None
|
| 273 |
+
restore_rng(checkpoint['rng'], generator)
|
| 274 |
+
run_dir = Path(resume).resolve().parent
|
| 275 |
+
del checkpoint
|
| 276 |
+
# Preserve this continuation's actual source/config alongside the
|
| 277 |
+
# original run, without overwriting its v2 source or provenance.
|
| 278 |
+
segment_dir = (run_dir / 'resume_segments'
|
| 279 |
+
/ f'{datetime.now(ZoneInfo("Europe/London")).strftime("%Y-%m-%d_%H-%M-%S_%f")}_{Path(__file__).stem}')
|
| 280 |
+
segment_dir.mkdir(parents=True, exist_ok=False)
|
| 281 |
+
shutil.copy2(__file__, segment_dir / Path(__file__).name)
|
| 282 |
+
shutil.copy2(model_bio.__file__, segment_dir / 'model_bio.py')
|
| 283 |
+
(segment_dir/'config.json').write_text(json.dumps(config,indent=2),encoding='utf-8')
|
| 284 |
+
continuation = {
|
| 285 |
+
'resume_checkpoint':str(Path(resume).resolve()),
|
| 286 |
+
'resume_checkpoint_sha256':digest(resume),
|
| 287 |
+
'start_step':state['step'],'epoch':state['epoch'],'offset':state['offset'],
|
| 288 |
+
'trainer_source':Path(__file__).name,'trainer_sha256':digest(__file__),
|
| 289 |
+
'model_source_sha256':digest(model_bio.__file__),
|
| 290 |
+
'configuration_source':'Saved resume checkpoint configuration',
|
| 291 |
+
'optimizations':OPTIMIZATIONS,
|
| 292 |
+
}
|
| 293 |
+
(segment_dir/'provenance.json').write_text(json.dumps(continuation,indent=2),encoding='utf-8')
|
| 294 |
+
state.setdefault('resume_segments',[]).append(str(segment_dir.relative_to(run_dir)))
|
| 295 |
+
else:
|
| 296 |
+
run_dir = ROOT / 'runs' / f'{Path(__file__).stem}_{datetime.now(ZoneInfo("Europe/London")).strftime("%Y-%m-%d_%H-%M-%S")}_{task}'
|
| 297 |
+
run_dir.mkdir(parents=True, exist_ok=False)
|
| 298 |
+
state = {'step':0, 'epoch':0, 'offset':0, 'best_accuracy':-1.0, 'best_loss':float('inf'),
|
| 299 |
+
'best_step':0, 'nonfinite_updates':0, 'consecutive_nonfinite':0, 'last_eval_step':-1,
|
| 300 |
+
'elapsed_seconds':0.0, 'lr_scale':1.0, 'packed_loss_verification':None}
|
| 301 |
+
order = None
|
| 302 |
+
(run_dir / 'config.json').write_text(json.dumps(config, indent=2), encoding='utf-8')
|
| 303 |
+
shutil.copy2(__file__, run_dir / Path(__file__).name)
|
| 304 |
+
shutil.copy2(model_bio.__file__, run_dir / 'model_bio.py')
|
| 305 |
+
provenance = {'model_source_sha256':digest(model_bio.__file__), 'trainer_sha256':digest(__file__),
|
| 306 |
+
'starting_checkpoint':metadata, 'torch':torch.__version__, 'markers':MARKERS,
|
| 307 |
+
'native_backbone_dropout':base.config.dropout,
|
| 308 |
+
'data':{split:d['metadata'] for split,d in data.items()},
|
| 309 |
+
'all_backbone_parameters_train':True, 'optimizations':OPTIMIZATIONS}
|
| 310 |
+
(run_dir/'provenance.json').write_text(json.dumps(provenance, indent=2), encoding='utf-8')
|
| 311 |
+
state.setdefault('adaptive_checkpointing_microbatches',{'enabled':0,'disabled':0})
|
| 312 |
+
log = (run_dir / (run_dir.name + '.log')).open('a', encoding='utf-8', buffering=1)
|
| 313 |
+
def emit(event):
|
| 314 |
+
event.update(task=task)
|
| 315 |
+
line = json.dumps(event, ensure_ascii=False)
|
| 316 |
+
print(line, flush=True)
|
| 317 |
+
log.write(line + '\n')
|
| 318 |
+
with (run_dir/'metrics.jsonl').open('a',encoding='utf-8') as stream:
|
| 319 |
+
stream.write(line + '\n')
|
| 320 |
+
def save(name):
|
| 321 |
+
atomic_save(run_dir/name, {'model':model.state_dict(), 'optimizer':optimizer.state_dict(),
|
| 322 |
+
'state':dict(state), 'order':order, 'rng':rng_state(generator), 'config':config,
|
| 323 |
+
'task':task, 'model_args':vars(base.config), 'markers':MARKERS,
|
| 324 |
+
'source_hashes':{'model':digest(model_bio.__file__),'trainer':digest(__file__)}})
|
| 325 |
+
evaluation_batch = 16 if task == 'sst2' else 4
|
| 326 |
+
heldout = 'validation' if task == 'sst2' else 'test'
|
| 327 |
+
before_path = run_dir/'before.json'
|
| 328 |
+
start_time = time.perf_counter()
|
| 329 |
+
elapsed_prior = state['elapsed_seconds']
|
| 330 |
+
emit({'event':'start','step':state['step'],'train_examples':n_train,'epochs':config['epochs'],
|
| 331 |
+
'total_steps':total_steps,'microbatch':task_config['microbatch_size'],
|
| 332 |
+
'effective_batch':config['effective_batch_size'],'memory_cap_gib':round(cap/1024**3,2),
|
| 333 |
+
'trainable_parameters':sum(p.numel() for p in model.parameters() if p.requires_grad),
|
| 334 |
+
'device':torch.cuda.get_device_name(0), 'optimizations':OPTIMIZATIONS,
|
| 335 |
+
'verify_packed_loss_once':verify_packed_loss_once})
|
| 336 |
+
if not before_path.exists():
|
| 337 |
+
before = evaluate(model, data[heldout], evaluation_batch)
|
| 338 |
+
before_path.write_text(json.dumps(before,indent=2),encoding='utf-8')
|
| 339 |
+
emit({'event':'before','heldout':heldout,**full_summary(before)})
|
| 340 |
+
else:
|
| 341 |
+
before = json.loads(before_path.read_text(encoding='utf-8'))
|
| 342 |
+
train_fixed = torch.arange(min(128,n_train), device='cuda:0')
|
| 343 |
+
def evaluate_and_save():
|
| 344 |
+
validation = evaluate(model, data['validation'], evaluation_batch)
|
| 345 |
+
train_eval = evaluate(model, data['train'], evaluation_batch, train_fixed)
|
| 346 |
+
changed = (validation['accuracy'] > state['best_accuracy'] or
|
| 347 |
+
(validation['accuracy'] == state['best_accuracy'] and validation['loss'] < state['best_loss']))
|
| 348 |
+
if changed:
|
| 349 |
+
state['best_accuracy'] = validation['accuracy']
|
| 350 |
+
state['best_loss'] = validation['loss']
|
| 351 |
+
state['best_step'] = state['step']
|
| 352 |
+
state['last_eval_step'] = state['step']
|
| 353 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter() - start_time
|
| 354 |
+
# Once-per-evaluation diagnostic over twelve oscillator tensors, not Python-built training batches.
|
| 355 |
+
omegas = torch.cat([p.detach().reshape(-1) for name,p in model.named_parameters() if name.endswith('.omega')]) # lint: ok STACK-COMPREHENSION once-per-evaluation statistics over twelve oscillator tensors
|
| 356 |
+
emit({'event':'eval','step':state['step'],'epoch':state['epoch'],'offset':state['offset'],
|
| 357 |
+
'train':full_summary(train_eval),'validation':full_summary(validation),
|
| 358 |
+
'best_step':state['best_step'],'omega_std':float(omegas.std(correction=0)),
|
| 359 |
+
'peak_allocated_gib':round(torch.cuda.max_memory_allocated()/1024**3,3),
|
| 360 |
+
'sample':{'gold':int(data['validation']['labels'][0]),'predicted':validation['predictions'][0]},
|
| 361 |
+
'elapsed_seconds':round(state['elapsed_seconds'],1)})
|
| 362 |
+
if changed:
|
| 363 |
+
save('best.pt')
|
| 364 |
+
save('latest.pt')
|
| 365 |
+
model.train()
|
| 366 |
+
if state['last_eval_step'] < 0:
|
| 367 |
+
evaluate_and_save()
|
| 368 |
+
model.train()
|
| 369 |
+
accumulated = torch.zeros(3,device='cuda:0')
|
| 370 |
+
logged_steps = 0
|
| 371 |
+
log_start = time.perf_counter()
|
| 372 |
+
stopping_cause = 'epochs_complete'
|
| 373 |
+
while state['epoch'] < config['epochs']:
|
| 374 |
+
if order is None:
|
| 375 |
+
order = torch.randperm(n_train,device='cuda:0',generator=generator)
|
| 376 |
+
# Reconstructed identically from saved order/CPU lengths after resume;
|
| 377 |
+
# no extra RNG draws and no per-microbatch CUDA-to-CPU length transfers.
|
| 378 |
+
microbatch_plan = plan_epoch_microbatches(
|
| 379 |
+
data['train'], order, config['effective_batch_size'], task_config['microbatch_size'])
|
| 380 |
+
while state['offset'] < n_train:
|
| 381 |
+
end = min(n_train,state['offset']+config['effective_batch_size'])
|
| 382 |
+
indices = order[state['offset']:end]
|
| 383 |
+
effective = len(indices)
|
| 384 |
+
lr = schedule(state['step'], total_steps, config) * state['lr_scale']
|
| 385 |
+
for group in optimizer.param_groups:
|
| 386 |
+
group['lr'] = lr
|
| 387 |
+
optimizer.zero_grad(set_to_none=True)
|
| 388 |
+
batch_values = torch.zeros(3,device='cuda:0')
|
| 389 |
+
for position in range(0,effective,task_config['microbatch_size']):
|
| 390 |
+
micro = indices[position:position+task_config['microbatch_size']]
|
| 391 |
+
width, packed_lm_tokens = microbatch_plan[state['offset'] + position]
|
| 392 |
+
x,lengths,labels = select_batch(data['train'],micro,width)
|
| 393 |
+
if task != 'sst2':
|
| 394 |
+
base.config.gradient_checkpointing = width > CHECKPOINT_THRESHOLD
|
| 395 |
+
checkpoint_key = 'enabled' if base.config.gradient_checkpointing else 'disabled'
|
| 396 |
+
state['adaptive_checkpointing_microbatches'][checkpoint_key] += 1
|
| 397 |
+
verify_this = (verify_packed_loss_once and task != 'sst2'
|
| 398 |
+
and not state.get('packed_loss_verification'))
|
| 399 |
+
# AMP-COMPLEX: real-valued model; native rotary complex operations remain explicitly float32.
|
| 400 |
+
with torch.autocast('cuda',dtype=torch.bfloat16): # lint: ok AMP-COMPLEX real native weights; RoPE explicitly uses float32 complex rotation
|
| 401 |
+
logits,lm_loss = model(x,lengths,auxiliary=True,
|
| 402 |
+
packed_lm_tokens=packed_lm_tokens,verify_packed=verify_this)
|
| 403 |
+
classification_loss = F.cross_entropy(logits,labels)
|
| 404 |
+
loss = classification_loss + config['lm_coefficient'] * lm_loss
|
| 405 |
+
weighted = loss * (len(micro)/effective)
|
| 406 |
+
if verify_this:
|
| 407 |
+
verification = model.last_loss_verification
|
| 408 |
+
state['packed_loss_verification'] = verification
|
| 409 |
+
(run_dir/'packed_loss_verification.json').write_text(
|
| 410 |
+
json.dumps(verification,indent=2),encoding='utf-8')
|
| 411 |
+
emit({'event':'packed_loss_verification','step':state['step'],**verification})
|
| 412 |
+
if not verification['passed']:
|
| 413 |
+
raise RuntimeError('Packed auxiliary LM loss did not match dense masked loss; no optimizer update made.')
|
| 414 |
+
weighted.backward()
|
| 415 |
+
batch_values += torch.stack((loss.detach(),classification_loss.detach(),lm_loss.detach())) * (len(micro)/effective)
|
| 416 |
+
norm = torch.nn.utils.clip_grad_norm_(model.parameters(),config['grad_clip'])
|
| 417 |
+
if torch.isfinite(norm):
|
| 418 |
+
optimizer.step()
|
| 419 |
+
state['consecutive_nonfinite'] = 0
|
| 420 |
+
else:
|
| 421 |
+
state['nonfinite_updates'] += 1
|
| 422 |
+
state['consecutive_nonfinite'] += 1
|
| 423 |
+
if state['consecutive_nonfinite'] >= 2:
|
| 424 |
+
state['lr_scale'] *= 0.5
|
| 425 |
+
emit({'event':'nonfinite_update_skipped','step':state['step']+1,
|
| 426 |
+
'count':state['nonfinite_updates'],'lr_scale':state['lr_scale']})
|
| 427 |
+
state['step'] += 1
|
| 428 |
+
state['offset'] = end
|
| 429 |
+
accumulated += batch_values
|
| 430 |
+
logged_steps += 1
|
| 431 |
+
if state['step'] == 1 or state['step'] % config['log_interval'] == 0:
|
| 432 |
+
# One host transfer per logging interval; hot microbatches remain on CUDA.
|
| 433 |
+
values = (accumulated/logged_steps).cpu().tolist() # lint: ok SYNC-IN-LOOP once per logging interval
|
| 434 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter()-start_time
|
| 435 |
+
emit({'event':'train','step':state['step'],'total_steps':total_steps,
|
| 436 |
+
'epoch_fraction':round(state['epoch']+end/n_train,4),'loss':values[0],
|
| 437 |
+
'classification_loss':values[1],'lm_loss':values[2],'lr':lr,
|
| 438 |
+
'updates_per_second':round(logged_steps/(time.perf_counter()-log_start),3),
|
| 439 |
+
'peak_allocated_gib':round(torch.cuda.max_memory_allocated()/1024**3,3),
|
| 440 |
+
'nonfinite_updates':state['nonfinite_updates'],'elapsed_seconds':round(state['elapsed_seconds'],1)})
|
| 441 |
+
accumulated.zero_(); logged_steps=0;log_start=time.perf_counter()
|
| 442 |
+
if state['step'] % task_config['eval_interval'] == 0:
|
| 443 |
+
evaluate_and_save()
|
| 444 |
+
log_start=time.perf_counter()
|
| 445 |
+
elif state['step'] % config['checkpoint_interval'] == 0:
|
| 446 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter()-start_time
|
| 447 |
+
save('latest.pt')
|
| 448 |
+
if STOP or state['consecutive_nonfinite'] >= config['max_consecutive_nonfinite']:
|
| 449 |
+
stopping_cause = 'user_interrupt' if STOP else 'nonfinite_limit'
|
| 450 |
+
state['elapsed_seconds'] = elapsed_prior + time.perf_counter()-start_time
|
| 451 |
+
save('latest.pt')
|
| 452 |
+
emit({'event':'stopped','stopping_cause':stopping_cause,'step':state['step']})
|
| 453 |
+
log.close()
|
| 454 |
+
return False
|
| 455 |
+
state['epoch'] += 1
|
| 456 |
+
state['offset'] = 0
|
| 457 |
+
order = None
|
| 458 |
+
if state['last_eval_step'] != state['step']:
|
| 459 |
+
evaluate_and_save()
|
| 460 |
+
else:
|
| 461 |
+
save('latest.pt')
|
| 462 |
+
# Final held-out scores use the best validation-selected checkpoint.
|
| 463 |
+
best = torch.load(run_dir/'best.pt',map_location='cpu',weights_only=False,mmap=True)
|
| 464 |
+
model.load_state_dict(best['model'],strict=True)
|
| 465 |
+
best_step = best['state']['step']
|
| 466 |
+
del best
|
| 467 |
+
after = evaluate(model,data[heldout],evaluation_batch)
|
| 468 |
+
(run_dir/'after.json').write_text(json.dumps(after,indent=2),encoding='utf-8')
|
| 469 |
+
rows = raw_rows(task,heldout)
|
| 470 |
+
examples = []
|
| 471 |
+
for row_index,(row,pb,pa) in enumerate(zip(rows,before['predictions'],after['predictions'])):
|
| 472 |
+
example = {'id':f'{task}/{heldout}/{row_index}','input':row.get('text',row.get('article','')),
|
| 473 |
+
'prompt':row.get('question',''),'options':row.get('options',['negative','positive']),
|
| 474 |
+
'gold':row['label'],'predicted_before':pb,'predicted_after':pa,
|
| 475 |
+
'correct_before':pb==row['label'],'correct_after':pa==row['label']}
|
| 476 |
+
examples.append(example)
|
| 477 |
+
(run_dir/'heldout_predictions.json').write_text(json.dumps(examples,ensure_ascii=False,indent=2),encoding='utf-8')
|
| 478 |
+
probe_count = 100 if task=='sst2' else 40
|
| 479 |
+
probe = {'before_correct':sum(e['correct_before'] for e in examples[:probe_count]),
|
| 480 |
+
'after_correct':sum(e['correct_after'] for e in examples[:probe_count]),'total':probe_count}
|
| 481 |
+
state['elapsed_seconds'] = elapsed_prior+time.perf_counter()-start_time
|
| 482 |
+
summary = {'id':task,'name':'SST-2 sentiment' if task=='sst2' else 'RACE middle-school reading',
|
| 483 |
+
'train_examples':n_train,'validation_examples':len(data['validation']['labels']),
|
| 484 |
+
'test_examples':len(data['test']['labels']) if task!='sst2' else 1821,
|
| 485 |
+
'heldout_split':heldout,'baseline':{'label':'Before task training (same classifier)',**full_summary(before)},
|
| 486 |
+
'finetuned':full_summary(after),'chance_accuracy':0.5 if task=='sst2' else 0.25,
|
| 487 |
+
'majority_accuracy':after['majority_accuracy'],'paired_earlier_probe':probe,
|
| 488 |
+
'training':{'epochs':config['epochs'],'max_steps':total_steps,'steps':state['step'],
|
| 489 |
+
'batch_size':config['effective_batch_size'],'microbatch_size':task_config['microbatch_size'],
|
| 490 |
+
'gradient_accumulation':config['effective_batch_size']//task_config['microbatch_size'],
|
| 491 |
+
'learning_rate':config['learning_rate'],'max_length':config['max_length'],
|
| 492 |
+
'lm_coefficient':config['lm_coefficient'],'stopping_cause':stopping_cause,
|
| 493 |
+
'classifier_dropout':config['classifier_dropout'],'native_backbone_dropout':base.config.dropout,
|
| 494 |
+
'nonfinite_updates':state['nonfinite_updates'],'best_step':best_step,
|
| 495 |
+
'peak_allocated_gib':round(torch.cuda.max_memory_allocated()/1024**3,3),
|
| 496 |
+
'trainable_parameters':sum(p.numel() for p in model.parameters()),
|
| 497 |
+
'head':'768-to-2 linear' if task=='sst2' else 'shared 768-to-1 linear over four answers',
|
| 498 |
+
'checkpoint_selection':'highest validation accuracy, lower validation classification loss breaks ties',
|
| 499 |
+
'optimizations':OPTIMIZATIONS,
|
| 500 |
+
'adaptive_checkpointing_microbatches':state['adaptive_checkpointing_microbatches'],
|
| 501 |
+
'resume_segments':state.get('resume_segments',[]),
|
| 502 |
+
'packed_loss_verification':state.get('packed_loss_verification'),
|
| 503 |
+
'details':'Full native backbone trained. Zero decay on oscillator and 1-D parameters. Linear GPT-1 warmup/decay, AdamW, gradient clip1.0. Original backbone dropout retained; classifier dropout0.1. Three unused padding-vocabulary slots initialized for task markers. V2 padding cropping and packed LM projection retained. V3 RACE native gradient checkpointing is enabled above 384 cropped tokens and disabled at or below 384; counters cover v3 segments, while earlier v2 segments used checkpointing throughout. Question order, effective batch and candidate-wise loss normalization are preserved.'},
|
| 504 |
+
'examples':examples,'checkpoint':{'name':str((run_dir/'best.pt').relative_to(ROOT)),
|
| 505 |
+
'sha256':digest(run_dir/'best.pt')},
|
| 506 |
+
'elapsed_seconds':state['elapsed_seconds']}
|
| 507 |
+
(run_dir/'summary.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2),encoding='utf-8')
|
| 508 |
+
emit({'event':'complete','step':state['step'],'best_step':best_step,'stopping_cause':stopping_cause,
|
| 509 |
+
'heldout':heldout,'before':full_summary(before),'after':full_summary(after),
|
| 510 |
+
'paired_probe':probe,'elapsed_seconds':round(state['elapsed_seconds'],1)})
|
| 511 |
+
log.close()
|
| 512 |
+
del model,base,optimizer,data
|
| 513 |
+
gc.collect();torch.cuda.empty_cache()
|
| 514 |
+
return True
|
| 515 |
+
|
| 516 |
+
def write_comparison():
|
| 517 |
+
tasks=[]
|
| 518 |
+
for task in ['sst2','race_middle']:
|
| 519 |
+
paths=sorted((ROOT/'runs').glob(f'*_{task}/summary.json'),key=lambda path:path.stat().st_mtime)
|
| 520 |
+
if not paths:
|
| 521 |
+
return
|
| 522 |
+
tasks.append(json.loads(paths[-1].read_text(encoding='utf-8')))
|
| 523 |
+
data={'date':datetime.now(ZoneInfo('Europe/London')).isoformat(),
|
| 524 |
+
'scope':'Full SST-2 training and validation; full RACE-middle training, validation selection and held-out test. Historical GPT-1 figures use different training coverage and test sets.',
|
| 525 |
+
'method':'Two independent full-model adaptations from original pretrained WiggleGPT. Classifier cross-entropy plus0.5 times token-average auxiliary language-model cross-entropy, with GPT-1-style input markers and linear task heads.',
|
| 526 |
+
'control_method':'Identical task inputs and initialized classifier evaluated before task training; after scores use validation-selected trained checkpoint. Earlier language-model likelihood probes are separately identified.',
|
| 527 |
+
'historical':{'gpt1_race_accuracy':59.0,'gpt1_race_middle_accuracy':62.9,'gpt1_sst2_accuracy':91.3,
|
| 528 |
+
'source_url':'https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf',
|
| 529 |
+
'note':'GPT-1 SST-2 91.3% is GLUE test, while this run reports public validation. GPT-1 RACE-M62.9% used combined RACE-middle/high training, while Wiggle used middle-only. Historical rows give context and are not a matched-model replication.'},
|
| 530 |
+
'tasks':tasks,'elapsed_seconds':sum(t['elapsed_seconds'] for t in tasks)}
|
| 531 |
+
(ROOT/'comparison.json').write_text(json.dumps(data,ensure_ascii=False,indent=2),encoding='utf-8')
|
| 532 |
+
|
| 533 |
+
def main():
|
| 534 |
+
parser=argparse.ArgumentParser(description=__doc__)
|
| 535 |
+
parser.add_argument('--task',choices=['all','sst2','race_middle'],default='all')
|
| 536 |
+
parser.add_argument('--resume',type=Path)
|
| 537 |
+
parser.add_argument('--config',type=Path,default=ROOT/'train_config.json')
|
| 538 |
+
parser.add_argument('--starting-checkpoint',type=Path,help='Original pretrained checkpoint; overrides config for a new run.')
|
| 539 |
+
parser.add_argument('--verify-packed-loss-once',action='store_true',
|
| 540 |
+
help='Before the first RACE optimizer update, compare dense and packed auxiliary LM loss on one actual CUDA training candidate and record the result.')
|
| 541 |
+
args=parser.parse_args()
|
| 542 |
+
config=json.loads(args.config.read_text(encoding='utf-8'))
|
| 543 |
+
if args.starting_checkpoint:
|
| 544 |
+
config['starting_checkpoint']=str(args.starting_checkpoint.expanduser().resolve())
|
| 545 |
+
torch.set_num_threads(config['cpu_threads'])
|
| 546 |
+
try:
|
| 547 |
+
import psutil
|
| 548 |
+
psutil.Process().nice(psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == 'nt' else 10)
|
| 549 |
+
except (ImportError, OSError):
|
| 550 |
+
print('Optional process priority setting unavailable; continuing with configured CPU threads.',flush=True)
|
| 551 |
+
if not torch.cuda.is_available():
|
| 552 |
+
raise RuntimeError('CUDA required; CPU model execution is disabled.')
|
| 553 |
+
torch.cuda.set_device(0)
|
| 554 |
+
torch.set_float32_matmul_precision('high')
|
| 555 |
+
signal.signal(signal.SIGINT,stop_requested)
|
| 556 |
+
if args.resume:
|
| 557 |
+
resume_meta=torch.load(args.resume,map_location='cpu',weights_only=False,mmap=True)
|
| 558 |
+
selected=[resume_meta['task']]
|
| 559 |
+
config=resume_meta['config']
|
| 560 |
+
torch.set_num_threads(config['cpu_threads'])
|
| 561 |
+
del resume_meta
|
| 562 |
+
else:
|
| 563 |
+
selected=['sst2','race_middle'] if args.task=='all' else [args.task]
|
| 564 |
+
for task in selected:
|
| 565 |
+
if not run_task(task,config,args.resume,args.verify_packed_loss_once):
|
| 566 |
+
return
|
| 567 |
+
write_comparison()
|
| 568 |
+
print('All requested task adaptations complete.',flush=True)
|
| 569 |
+
|
| 570 |
+
if __name__=='__main__':
|
| 571 |
+
main()
|