| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.019843238416509576, |
| "eval_steps": 500, |
| "global_step": 100, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "completion_length": 673.5, |
| "epoch": 0.00019843238416509574, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "step": 1 |
| }, |
| { |
| "completion_length": 589.625, |
| "epoch": 0.0003968647683301915, |
| "grad_norm": 0.2197265625, |
| "kl": 0.0, |
| "learning_rate": 1e-05, |
| "loss": -0.0, |
| "reward": 1.375, |
| "reward_std": 0.39433756470680237, |
| "rewards/correctness_reward_func": 1.375, |
| "step": 2 |
| }, |
| { |
| "completion_length": 408.28125, |
| "epoch": 0.0005952971524952872, |
| "grad_norm": 0.1533203125, |
| "kl": 0.00010921969078481197, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.4375, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.4375, |
| "step": 3 |
| }, |
| { |
| "completion_length": 606.40625, |
| "epoch": 0.000793729536660383, |
| "grad_norm": 0.193359375, |
| "kl": 0.0011470153476693667, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.625, |
| "reward_std": 0.25, |
| "rewards/correctness_reward_func": 0.625, |
| "step": 4 |
| }, |
| { |
| "completion_length": 382.59375, |
| "epoch": 0.0009921619208254787, |
| "grad_norm": 0.18359375, |
| "kl": 0.0041184365400113165, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.3125, |
| "reward_std": 0.26933756470680237, |
| "rewards/correctness_reward_func": 1.3125, |
| "step": 5 |
| }, |
| { |
| "completion_length": 528.28125, |
| "epoch": 0.0011905943049905744, |
| "grad_norm": 0.002655029296875, |
| "kl": 0.0031762155922478996, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.5, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 1.5, |
| "step": 6 |
| }, |
| { |
| "completion_length": 486.625, |
| "epoch": 0.0013890266891556703, |
| "grad_norm": 0.22265625, |
| "kl": 0.006348274531774223, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.625, |
| "reward_std": 0.14433756470680237, |
| "rewards/correctness_reward_func": 1.625, |
| "step": 7 |
| }, |
| { |
| "completion_length": 295.09375, |
| "epoch": 0.001587459073320766, |
| "grad_norm": 0.1318359375, |
| "kl": 0.00916035019326955, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.6875, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.6875, |
| "step": 8 |
| }, |
| { |
| "completion_length": 305.9375, |
| "epoch": 0.0017858914574858616, |
| "grad_norm": 0.003814697265625, |
| "kl": 0.01111468207091093, |
| "learning_rate": 1e-05, |
| "loss": 0.0001, |
| "reward": 2.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 2.0, |
| "step": 9 |
| }, |
| { |
| "completion_length": 262.59375, |
| "epoch": 0.0019843238416509573, |
| "grad_norm": 0.006805419921875, |
| "kl": 0.01166562782600522, |
| "learning_rate": 1e-05, |
| "loss": 0.0001, |
| "reward": 2.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 2.0, |
| "step": 10 |
| }, |
| { |
| "completion_length": 427.28125, |
| "epoch": 0.002182756225816053, |
| "grad_norm": 0.134765625, |
| "kl": 0.0021403132705017924, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.4375, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.4375, |
| "step": 11 |
| }, |
| { |
| "completion_length": 454.96875, |
| "epoch": 0.0023811886099811487, |
| "grad_norm": 0.15625, |
| "kl": 0.009784933528862894, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.1875, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.1875, |
| "step": 12 |
| }, |
| { |
| "completion_length": 520.0625, |
| "epoch": 0.002579620994146245, |
| "grad_norm": 0.208984375, |
| "kl": 0.0027196165174245834, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.8125, |
| "reward_std": 0.375, |
| "rewards/correctness_reward_func": 0.8125, |
| "step": 13 |
| }, |
| { |
| "completion_length": 270.5625, |
| "epoch": 0.0027780533783113405, |
| "grad_norm": 0.1396484375, |
| "kl": 0.00326572876656428, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.1875, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.1875, |
| "step": 14 |
| }, |
| { |
| "completion_length": 302.71875, |
| "epoch": 0.0029764857624764362, |
| "grad_norm": 0.09130859375, |
| "kl": 0.0021118283621035516, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.9375, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.9375, |
| "step": 15 |
| }, |
| { |
| "completion_length": 289.03125, |
| "epoch": 0.003174918146641532, |
| "grad_norm": 0.123046875, |
| "kl": 0.006330179603537545, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.9375, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.9375, |
| "step": 16 |
| }, |
| { |
| "completion_length": 377.28125, |
| "epoch": 0.0033733505308066276, |
| "grad_norm": 0.0027923583984375, |
| "kl": 0.008314917489769869, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.5, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 1.5, |
| "step": 17 |
| }, |
| { |
| "completion_length": 366.1875, |
| "epoch": 0.0035717829149717233, |
| "grad_norm": 0.0023956298828125, |
| "kl": 0.006974527728743851, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.25, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 1.25, |
| "step": 18 |
| }, |
| { |
| "completion_length": 455.53125, |
| "epoch": 0.003770215299136819, |
| "grad_norm": 0.1455078125, |
| "kl": 0.0025141297810478136, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.25, |
| "reward_std": 0.39433756470680237, |
| "rewards/correctness_reward_func": 1.25, |
| "step": 19 |
| }, |
| { |
| "completion_length": 512.03125, |
| "epoch": 0.003968647683301915, |
| "grad_norm": 0.1826171875, |
| "kl": 0.0014448043148149736, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.1875, |
| "reward_std": 0.26933756470680237, |
| "rewards/correctness_reward_func": 1.1875, |
| "step": 20 |
| }, |
| { |
| "completion_length": 539.78125, |
| "epoch": 0.00416708006746701, |
| "grad_norm": 0.1328125, |
| "kl": 0.0007829267560737208, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.375, |
| "reward_std": 0.25, |
| "rewards/correctness_reward_func": 1.375, |
| "step": 21 |
| }, |
| { |
| "completion_length": 524.8125, |
| "epoch": 0.004365512451632106, |
| "grad_norm": 0.1962890625, |
| "kl": 0.0011430769227445126, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.125, |
| "reward_std": 0.25, |
| "rewards/correctness_reward_func": 1.125, |
| "step": 22 |
| }, |
| { |
| "completion_length": 354.46875, |
| "epoch": 0.004563944835797202, |
| "grad_norm": 0.12890625, |
| "kl": 0.0008475587819702923, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.8125, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.8125, |
| "step": 23 |
| }, |
| { |
| "completion_length": 358.15625, |
| "epoch": 0.0047623772199622974, |
| "grad_norm": 0.00142669677734375, |
| "kl": 0.0014332281207316555, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 2.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 2.0, |
| "step": 24 |
| }, |
| { |
| "completion_length": 458.84375, |
| "epoch": 0.004960809604127394, |
| "grad_norm": 0.1318359375, |
| "kl": 0.0006973636773182079, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.5625, |
| "reward_std": 0.26933756470680237, |
| "rewards/correctness_reward_func": 1.5625, |
| "step": 25 |
| }, |
| { |
| "completion_length": 391.46875, |
| "epoch": 0.00515924198829249, |
| "grad_norm": 0.189453125, |
| "kl": 0.0014636927953688428, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.625, |
| "reward_std": 0.25, |
| "rewards/correctness_reward_func": 1.625, |
| "step": 26 |
| }, |
| { |
| "completion_length": 258.3125, |
| "epoch": 0.005357674372457585, |
| "grad_norm": 0.1484375, |
| "kl": 0.004023743560537696, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.9375, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.9375, |
| "step": 27 |
| }, |
| { |
| "completion_length": 271.75, |
| "epoch": 0.005556106756622681, |
| "grad_norm": 0.12109375, |
| "kl": 0.0016350722071365453, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.9375, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.9375, |
| "step": 28 |
| }, |
| { |
| "completion_length": 308.96875, |
| "epoch": 0.005754539140787777, |
| "grad_norm": 0.001739501953125, |
| "kl": 0.003819223667960614, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.5, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 1.5, |
| "step": 29 |
| }, |
| { |
| "completion_length": 515.9375, |
| "epoch": 0.0059529715249528724, |
| "grad_norm": 0.2451171875, |
| "kl": 0.0021796236469526775, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.75, |
| "reward_std": 0.39433756470680237, |
| "rewards/correctness_reward_func": 0.75, |
| "step": 30 |
| }, |
| { |
| "completion_length": 403.65625, |
| "epoch": 0.006151403909117968, |
| "grad_norm": 0.140625, |
| "kl": 0.0013396024442045018, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.5, |
| "reward_std": 0.25, |
| "rewards/correctness_reward_func": 1.5, |
| "step": 31 |
| }, |
| { |
| "completion_length": 469.875, |
| "epoch": 0.006349836293283064, |
| "grad_norm": 0.1455078125, |
| "kl": 0.001866564794909209, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.5, |
| "reward_std": 0.25, |
| "rewards/correctness_reward_func": 1.5, |
| "step": 32 |
| }, |
| { |
| "completion_length": 472.1875, |
| "epoch": 0.0065482686774481595, |
| "grad_norm": 0.1103515625, |
| "kl": 0.0019174512126483023, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.0625, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.0625, |
| "step": 33 |
| }, |
| { |
| "completion_length": 278.5625, |
| "epoch": 0.006746701061613255, |
| "grad_norm": 0.0005950927734375, |
| "kl": 0.0010064128291560337, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.75, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 1.75, |
| "step": 34 |
| }, |
| { |
| "completion_length": 391.03125, |
| "epoch": 0.006945133445778351, |
| "grad_norm": 0.09375, |
| "kl": 0.0012205626408103853, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.4375, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.4375, |
| "step": 35 |
| }, |
| { |
| "completion_length": 433.625, |
| "epoch": 0.007143565829943447, |
| "grad_norm": 0.189453125, |
| "kl": 0.0007209242321550846, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.8125, |
| "reward_std": 0.375, |
| "rewards/correctness_reward_func": 1.8125, |
| "step": 36 |
| }, |
| { |
| "completion_length": 362.71875, |
| "epoch": 0.007341998214108542, |
| "grad_norm": 0.11474609375, |
| "kl": 0.0012260141083970666, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.875, |
| "reward_std": 0.14433756470680237, |
| "rewards/correctness_reward_func": 1.875, |
| "step": 37 |
| }, |
| { |
| "completion_length": 339.875, |
| "epoch": 0.007540430598273638, |
| "grad_norm": 0.0027618408203125, |
| "kl": 0.009935397887602448, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.75, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 1.75, |
| "step": 38 |
| }, |
| { |
| "completion_length": 379.5, |
| "epoch": 0.007738862982438734, |
| "grad_norm": 0.185546875, |
| "kl": 0.0012411218194756657, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.9375, |
| "reward_std": 0.26933756470680237, |
| "rewards/correctness_reward_func": 0.9375, |
| "step": 39 |
| }, |
| { |
| "completion_length": 555.34375, |
| "epoch": 0.00793729536660383, |
| "grad_norm": 0.1201171875, |
| "kl": 0.007853956165490672, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.0625, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.0625, |
| "step": 40 |
| }, |
| { |
| "completion_length": 624.5, |
| "epoch": 0.008135727750768925, |
| "grad_norm": 0.279296875, |
| "kl": 0.0027262063522357494, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.8125, |
| "reward_std": 0.5193375647068024, |
| "rewards/correctness_reward_func": 0.8125, |
| "step": 41 |
| }, |
| { |
| "completion_length": 525.125, |
| "epoch": 0.00833416013493402, |
| "grad_norm": 0.166015625, |
| "kl": 0.0035142535489285365, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.375, |
| "reward_std": 0.39433756470680237, |
| "rewards/correctness_reward_func": 1.375, |
| "step": 42 |
| }, |
| { |
| "completion_length": 456.65625, |
| "epoch": 0.008532592519099116, |
| "grad_norm": 0.000522613525390625, |
| "kl": 0.0011033852133550681, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.75, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.75, |
| "step": 43 |
| }, |
| { |
| "completion_length": 252.8125, |
| "epoch": 0.008731024903264212, |
| "grad_norm": 0.126953125, |
| "kl": 0.0013557878410210833, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.875, |
| "reward_std": 0.14433756470680237, |
| "rewards/correctness_reward_func": 1.875, |
| "step": 44 |
| }, |
| { |
| "completion_length": 283.5625, |
| "epoch": 0.008929457287429308, |
| "grad_norm": 0.00055694580078125, |
| "kl": 0.0010409851965960115, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 1.0, |
| "step": 45 |
| }, |
| { |
| "completion_length": 452.6875, |
| "epoch": 0.009127889671594403, |
| "grad_norm": 0.0810546875, |
| "kl": 0.0016598845832049847, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.8125, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.8125, |
| "step": 46 |
| }, |
| { |
| "completion_length": 504.59375, |
| "epoch": 0.0093263220557595, |
| "grad_norm": 0.1064453125, |
| "kl": 0.001195055043353932, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.6875, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.6875, |
| "step": 47 |
| }, |
| { |
| "completion_length": 350.375, |
| "epoch": 0.009524754439924595, |
| "grad_norm": 0.06591796875, |
| "kl": 0.0007179172534961253, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.9375, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.9375, |
| "step": 48 |
| }, |
| { |
| "completion_length": 356.75, |
| "epoch": 0.009723186824089692, |
| "grad_norm": 0.06640625, |
| "kl": 0.004019862040877342, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.9375, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.9375, |
| "step": 49 |
| }, |
| { |
| "completion_length": 343.9375, |
| "epoch": 0.009921619208254788, |
| "grad_norm": 0.00055694580078125, |
| "kl": 0.0014488446759060025, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 2.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 2.0, |
| "step": 50 |
| }, |
| { |
| "completion_length": 456.4375, |
| "epoch": 0.010120051592419884, |
| "grad_norm": 0.169921875, |
| "kl": 0.001344717078609392, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.5625, |
| "reward_std": 0.375, |
| "rewards/correctness_reward_func": 1.5625, |
| "step": 51 |
| }, |
| { |
| "completion_length": 489.3125, |
| "epoch": 0.01031848397658498, |
| "grad_norm": 0.1845703125, |
| "kl": 0.0024816631339490414, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.1875, |
| "reward_std": 0.375, |
| "rewards/correctness_reward_func": 1.1875, |
| "step": 52 |
| }, |
| { |
| "completion_length": 489.5625, |
| "epoch": 0.010516916360750075, |
| "grad_norm": 0.12451171875, |
| "kl": 0.016249713487923145, |
| "learning_rate": 1e-05, |
| "loss": 0.0001, |
| "reward": 0.5625, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 0.5625, |
| "step": 53 |
| }, |
| { |
| "completion_length": 556.8125, |
| "epoch": 0.01071534874491517, |
| "grad_norm": 0.2490234375, |
| "kl": 0.0019329865172039717, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.375, |
| "reward_std": 0.6443375647068024, |
| "rewards/correctness_reward_func": 1.375, |
| "step": 54 |
| }, |
| { |
| "completion_length": 514.0, |
| "epoch": 0.010913781129080266, |
| "grad_norm": 0.6796875, |
| "kl": 0.005961886083241552, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.6875, |
| "reward_std": 0.41367512941360474, |
| "rewards/correctness_reward_func": 0.6875, |
| "step": 55 |
| }, |
| { |
| "completion_length": 432.84375, |
| "epoch": 0.011112213513245362, |
| "grad_norm": 0.19140625, |
| "kl": 0.0009212676668539643, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.1875, |
| "reward_std": 0.41367512941360474, |
| "rewards/correctness_reward_func": 1.1875, |
| "step": 56 |
| }, |
| { |
| "completion_length": 434.0625, |
| "epoch": 0.011310645897410458, |
| "grad_norm": 0.318359375, |
| "kl": 0.0013161345414118841, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.0625, |
| "reward_std": 0.6636751294136047, |
| "rewards/correctness_reward_func": 1.0625, |
| "step": 57 |
| }, |
| { |
| "completion_length": 503.9375, |
| "epoch": 0.011509078281575554, |
| "grad_norm": 0.10791015625, |
| "kl": 0.0013501367502612993, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.6875, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 0.6875, |
| "step": 58 |
| }, |
| { |
| "completion_length": 577.25, |
| "epoch": 0.01170751066574065, |
| "grad_norm": 0.06298828125, |
| "kl": 0.0004623739223461598, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.1875, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 0.1875, |
| "step": 59 |
| }, |
| { |
| "completion_length": 639.375, |
| "epoch": 0.011905943049905745, |
| "grad_norm": 0.0908203125, |
| "kl": 0.0009281258899136446, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.6875, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 0.6875, |
| "step": 60 |
| }, |
| { |
| "completion_length": 631.3125, |
| "epoch": 0.01210437543407084, |
| "grad_norm": 0.000553131103515625, |
| "kl": 0.0021498738351510838, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.75, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.75, |
| "step": 61 |
| }, |
| { |
| "completion_length": 328.8125, |
| "epoch": 0.012302807818235936, |
| "grad_norm": 0.000690460205078125, |
| "kl": 0.0010117258789250627, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.75, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.75, |
| "step": 62 |
| }, |
| { |
| "completion_length": 273.8125, |
| "epoch": 0.012501240202401032, |
| "grad_norm": 0.103515625, |
| "kl": 0.000650071247946471, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.4375, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.4375, |
| "step": 63 |
| }, |
| { |
| "completion_length": 450.03125, |
| "epoch": 0.012699672586566128, |
| "grad_norm": 0.1552734375, |
| "kl": 0.0009097288202610798, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.8125, |
| "reward_std": 0.26933756470680237, |
| "rewards/correctness_reward_func": 1.8125, |
| "step": 64 |
| }, |
| { |
| "completion_length": 415.96875, |
| "epoch": 0.012898104970731223, |
| "grad_norm": 0.0830078125, |
| "kl": 0.0014095778460614383, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.875, |
| "reward_std": 0.14433756470680237, |
| "rewards/correctness_reward_func": 1.875, |
| "step": 65 |
| }, |
| { |
| "completion_length": 421.8125, |
| "epoch": 0.013096537354896319, |
| "grad_norm": 0.09130859375, |
| "kl": 0.0017693252448225394, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.9375, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.9375, |
| "step": 66 |
| }, |
| { |
| "completion_length": 322.65625, |
| "epoch": 0.013294969739061415, |
| "grad_norm": 0.1484375, |
| "kl": 0.007693824853049591, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.875, |
| "reward_std": 0.14433756470680237, |
| "rewards/correctness_reward_func": 1.875, |
| "step": 67 |
| }, |
| { |
| "completion_length": 273.0625, |
| "epoch": 0.01349340212322651, |
| "grad_norm": 0.1787109375, |
| "kl": 0.0014752715069334954, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.875, |
| "reward_std": 0.14433756470680237, |
| "rewards/correctness_reward_func": 1.875, |
| "step": 68 |
| }, |
| { |
| "completion_length": 252.09375, |
| "epoch": 0.013691834507391606, |
| "grad_norm": 0.11083984375, |
| "kl": 0.0045530187780968845, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.6875, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.6875, |
| "step": 69 |
| }, |
| { |
| "completion_length": 430.96875, |
| "epoch": 0.013890266891556702, |
| "grad_norm": 0.1953125, |
| "kl": 0.0021810558391734958, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.75, |
| "reward_std": 0.39433756470680237, |
| "rewards/correctness_reward_func": 1.75, |
| "step": 70 |
| }, |
| { |
| "completion_length": 539.40625, |
| "epoch": 0.014088699275721797, |
| "grad_norm": 0.10498046875, |
| "kl": 0.003831994123174809, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.6875, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 0.6875, |
| "step": 71 |
| }, |
| { |
| "completion_length": 557.53125, |
| "epoch": 0.014287131659886893, |
| "grad_norm": 0.002471923828125, |
| "kl": 0.006553357859957032, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 1.0, |
| "step": 72 |
| }, |
| { |
| "completion_length": 587.78125, |
| "epoch": 0.014485564044051989, |
| "grad_norm": 0.27734375, |
| "kl": 0.0071588484570384026, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.0, |
| "reward_std": 0.5386751294136047, |
| "rewards/correctness_reward_func": 1.0, |
| "step": 73 |
| }, |
| { |
| "completion_length": 496.125, |
| "epoch": 0.014683996428217085, |
| "grad_norm": 0.28125, |
| "kl": 0.005753638513851911, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.0625, |
| "reward_std": 0.375, |
| "rewards/correctness_reward_func": 1.0625, |
| "step": 74 |
| }, |
| { |
| "completion_length": 400.90625, |
| "epoch": 0.01488242881238218, |
| "grad_norm": 0.234375, |
| "kl": 0.002603573986561969, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.5, |
| "reward_std": 0.39433756470680237, |
| "rewards/correctness_reward_func": 1.5, |
| "step": 75 |
| }, |
| { |
| "completion_length": 499.75, |
| "epoch": 0.015080861196547276, |
| "grad_norm": 0.162109375, |
| "kl": 0.003349974052980542, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.3125, |
| "reward_std": 0.375, |
| "rewards/correctness_reward_func": 1.3125, |
| "step": 76 |
| }, |
| { |
| "completion_length": 421.53125, |
| "epoch": 0.015279293580712372, |
| "grad_norm": 0.359375, |
| "kl": 0.003231665992643684, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.375, |
| "reward_std": 0.5386751294136047, |
| "rewards/correctness_reward_func": 1.375, |
| "step": 77 |
| }, |
| { |
| "completion_length": 324.59375, |
| "epoch": 0.015477725964877467, |
| "grad_norm": 0.345703125, |
| "kl": 0.0044938469072803855, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.75, |
| "reward_std": 0.25, |
| "rewards/correctness_reward_func": 0.75, |
| "step": 78 |
| }, |
| { |
| "completion_length": 584.875, |
| "epoch": 0.015676158349042565, |
| "grad_norm": 0.0007781982421875, |
| "kl": 0.0039344872056972235, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.25, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.25, |
| "step": 79 |
| }, |
| { |
| "completion_length": 568.84375, |
| "epoch": 0.01587459073320766, |
| "grad_norm": 0.000675201416015625, |
| "kl": 0.0011096175148850307, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 1.0, |
| "step": 80 |
| }, |
| { |
| "completion_length": 402.15625, |
| "epoch": 0.016073023117372756, |
| "grad_norm": 0.0005645751953125, |
| "kl": 0.002124628212186508, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.75, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 1.75, |
| "step": 81 |
| }, |
| { |
| "completion_length": 303.84375, |
| "epoch": 0.01627145550153785, |
| "grad_norm": 0.000911712646484375, |
| "kl": 0.001547299834783189, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.5, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 1.5, |
| "step": 82 |
| }, |
| { |
| "completion_length": 442.40625, |
| "epoch": 0.016469887885702948, |
| "grad_norm": 0.22265625, |
| "kl": 0.0019549240387277678, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.6875, |
| "reward_std": 0.41367512941360474, |
| "rewards/correctness_reward_func": 1.6875, |
| "step": 83 |
| }, |
| { |
| "completion_length": 363.5625, |
| "epoch": 0.01666832026986804, |
| "grad_norm": 0.1005859375, |
| "kl": 0.0021169226529309526, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.4375, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.4375, |
| "step": 84 |
| }, |
| { |
| "completion_length": 333.5, |
| "epoch": 0.01686675265403314, |
| "grad_norm": 0.00101470947265625, |
| "kl": 0.0020763709326274693, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 2.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 2.0, |
| "step": 85 |
| }, |
| { |
| "completion_length": 260.6875, |
| "epoch": 0.017065185038198233, |
| "grad_norm": 0.2578125, |
| "kl": 0.0023609662894159555, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.9375, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.9375, |
| "step": 86 |
| }, |
| { |
| "completion_length": 317.78125, |
| "epoch": 0.01726361742236333, |
| "grad_norm": 0.1416015625, |
| "kl": 0.0023049223236739635, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.8125, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.8125, |
| "step": 87 |
| }, |
| { |
| "completion_length": 374.5625, |
| "epoch": 0.017462049806528424, |
| "grad_norm": 0.002838134765625, |
| "kl": 0.003084336465690285, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 2.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 2.0, |
| "step": 88 |
| }, |
| { |
| "completion_length": 400.75, |
| "epoch": 0.01766048219069352, |
| "grad_norm": 0.265625, |
| "kl": 0.004105731437448412, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.25, |
| "reward_std": 0.25, |
| "rewards/correctness_reward_func": 1.25, |
| "step": 89 |
| }, |
| { |
| "completion_length": 481.28125, |
| "epoch": 0.017858914574858616, |
| "grad_norm": 0.00104522705078125, |
| "kl": 0.004729912034235895, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.5, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.5, |
| "step": 90 |
| }, |
| { |
| "completion_length": 660.34375, |
| "epoch": 0.018057346959023713, |
| "grad_norm": 0.2421875, |
| "kl": 0.005954354302957654, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.0, |
| "reward_std": 0.6830126941204071, |
| "rewards/correctness_reward_func": 1.0, |
| "step": 91 |
| }, |
| { |
| "completion_length": 547.09375, |
| "epoch": 0.018255779343188807, |
| "grad_norm": 0.248046875, |
| "kl": 0.004669041052693501, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.625, |
| "reward_std": 0.6443375647068024, |
| "rewards/correctness_reward_func": 0.625, |
| "step": 92 |
| }, |
| { |
| "completion_length": 467.71875, |
| "epoch": 0.018454211727353904, |
| "grad_norm": 0.189453125, |
| "kl": 0.0035645719908643514, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.3125, |
| "reward_std": 0.375, |
| "rewards/correctness_reward_func": 1.3125, |
| "step": 93 |
| }, |
| { |
| "completion_length": 442.84375, |
| "epoch": 0.018652644111519, |
| "grad_norm": 0.146484375, |
| "kl": 0.0033500409917905927, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.3125, |
| "reward_std": 0.375, |
| "rewards/correctness_reward_func": 1.3125, |
| "step": 94 |
| }, |
| { |
| "completion_length": 457.875, |
| "epoch": 0.018851076495684096, |
| "grad_norm": 0.06787109375, |
| "kl": 0.0038696340925525874, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.8125, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.8125, |
| "step": 95 |
| }, |
| { |
| "completion_length": 522.15625, |
| "epoch": 0.01904950887984919, |
| "grad_norm": 0.1328125, |
| "kl": 0.005176389531698078, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.875, |
| "reward_std": 0.25, |
| "rewards/correctness_reward_func": 0.875, |
| "step": 96 |
| }, |
| { |
| "completion_length": 682.34375, |
| "epoch": 0.019247941264014287, |
| "grad_norm": 0.000698089599609375, |
| "kl": 0.003805596032179892, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "step": 97 |
| }, |
| { |
| "completion_length": 561.03125, |
| "epoch": 0.019446373648179385, |
| "grad_norm": 0.078125, |
| "kl": 0.003582520061172545, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.6875, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 0.6875, |
| "step": 98 |
| }, |
| { |
| "completion_length": 517.96875, |
| "epoch": 0.01964480603234448, |
| "grad_norm": 0.16796875, |
| "kl": 0.005401681060902774, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 0.0625, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 0.0625, |
| "step": 99 |
| }, |
| { |
| "completion_length": 377.4375, |
| "epoch": 0.019843238416509576, |
| "grad_norm": 0.12255859375, |
| "kl": 0.0034658076183404773, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "reward": 1.1875, |
| "reward_std": 0.125, |
| "rewards/correctness_reward_func": 1.1875, |
| "step": 100 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 1007800, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 200, |
| "save_steps": 10, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
