{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 2.5380304243394716, "eval_steps": 500, "global_step": 200, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.012810248198558846, "grad_norm": 0.011678964830935001, "learning_rate": 0.0, "loss": 0.0, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.234375, "rewards/strict_format_reward_func/std": 0.2514837086200714, "step": 1, "time_profile/curr_logprobs_and_update": 0.2617394689877983, "time_profile/old_logps": 33.7158753760159, "time_profile/ref_logps": 33.76804309990257, "time_profile/reward": 0.0047389911487698555, "time_profile/text_rollout": 474.3757369443774, "train/gen_step": 64.0, "und/advantages_abs_mean": 0.2001953125, "und/advantages_max": 2.125, "und/advantages_mean": 0.0, "und/advantages_min": -0.4375, "und/advantages_std": 0.2579461336135864, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.751953125, "und/completion/min_length": 250.0, "und/frac_reward_zero_std": 0.046875, "und/kl": 0.0, "und/loss": 0.0, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 107.0, "und/prompt/min_length": 52.0, "und/reward": 0.193359375, "und/reward_std": 0.28102907538414 }, { "epoch": 0.025620496397117692, "grad_norm": 0.00965250376611948, "learning_rate": 1e-05, "loss": 0.0, "step": 2, "time_profile/curr_logprobs_and_update": 0.26145937727415003, "train/gen_step": 128.0, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/kl": 0.0, "und/loss": 0.0 }, { "epoch": 0.03843074459567654, "grad_norm": 0.04503251612186432, "learning_rate": 1e-05, "loss": 0.0, "rewards/correctness_reward_func/mean": 0.4375, "rewards/correctness_reward_func/std": 0.8333333730697632, "rewards/strict_format_reward_func/mean": 0.15625, "rewards/strict_format_reward_func/std": 0.233588308095932, "step": 3, "time_profile/curr_logprobs_and_update": 0.2616513767570723, "time_profile/old_logps": 33.65838770195842, "time_profile/ref_logps": 33.75480777770281, "time_profile/reward": 0.004094251431524754, "time_profile/text_rollout": 504.14777049794793, "train/gen_step": 192.0, "und/advantages_abs_mean": 0.443603515625, "und/advantages_max": 2.1875, "und/advantages_mean": 0.0, "und/advantages_min": -1.6875, "und/advantages_std": 0.7080574631690979, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.79296875, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.421875, "und/kl": 0.00019532828423507453, "und/loss": 7.946068224740088e-06, "und/prompt/max_length": 247.0, "und/prompt/mean_length": 201.5, "und/prompt/min_length": 156.0, "und/reward": 0.5078125, "und/reward_std": 0.9087191820144653 }, { "epoch": 0.051240992794235385, "grad_norm": 0.04360778629779816, "learning_rate": 1e-05, "loss": 0.0001, "step": 4, "time_profile/curr_logprobs_and_update": 0.261664173391182, "train/gen_step": 256.0, "und/clip_ratio/high_mean": 0.0003149772164761089, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0003149772164761089, "und/kl": 0.00025711835689889995, "und/loss": 0.000590196544485444 }, { "epoch": 0.06405124099279423, "grad_norm": 0.005336050409823656, "learning_rate": 1e-05, "loss": 0.0001, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0625, "rewards/strict_format_reward_func/std": 0.1666666716337204, "step": 5, "time_profile/curr_logprobs_and_update": 0.2615119785768911, "time_profile/old_logps": 33.64620256703347, "time_profile/ref_logps": 33.72650748398155, "time_profile/reward": 0.004272850230336189, "time_profile/text_rollout": 491.0981372119859, "train/gen_step": 320.0, "und/advantages_abs_mean": 0.095703125, "und/advantages_max": 2.1875, "und/advantages_mean": 0.0, "und/advantages_min": -0.375, "und/advantages_std": 0.21501831710338593, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.9296875, "und/completion/min_length": 252.0, "und/frac_reward_zero_std": 0.5, "und/kl": 0.00318289036204078, "und/loss": 0.00012254955510471177, "und/prompt/max_length": 247.0, "und/prompt/mean_length": 152.5, "und/prompt/min_length": 58.0, "und/reward": 0.06640625, "und/reward_std": 0.23300214111804962 }, { "epoch": 0.07686148919135308, "grad_norm": 0.006930716801434755, "learning_rate": 1e-05, "loss": 0.0003, "step": 6, "time_profile/curr_logprobs_and_update": 0.26136891632631887, "train/gen_step": 384.0, "und/clip_ratio/high_mean": 0.0004957187084073666, "und/clip_ratio/low_mean": 0.00018175119475927204, "und/clip_ratio/region_mean": 0.0006774698958906811, "und/kl": 0.0036970916280552046, "und/loss": 5.486323918546532e-05 }, { "epoch": 0.08967173738991192, "grad_norm": 0.012152941897511482, "learning_rate": 1e-05, "loss": 0.0001, "rewards/correctness_reward_func/mean": 0.03125, "rewards/correctness_reward_func/std": 0.25, "rewards/strict_format_reward_func/mean": 0.15625, "rewards/strict_format_reward_func/std": 0.233588308095932, "step": 7, "time_profile/curr_logprobs_and_update": 0.26169058527739253, "time_profile/old_logps": 33.673401176929474, "time_profile/ref_logps": 33.76787843555212, "time_profile/reward": 0.00381280854344368, "time_profile/text_rollout": 473.39493833016604, "train/gen_step": 448.0, "und/advantages_abs_mean": 0.197509765625, "und/advantages_max": 2.0625, "und/advantages_mean": 0.0, "und/advantages_min": -0.5625, "und/advantages_std": 0.2547253966331482, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.755859375, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.0625, "und/kl": 0.0018817147374647902, "und/loss": 4.349416121840477e-05, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 107.0, "und/prompt/min_length": 52.0, "und/reward": 0.1689453125, "und/reward_std": 0.27497100830078125 }, { "epoch": 0.10248198558847077, "grad_norm": 0.011709543876349926, "learning_rate": 1e-05, "loss": 0.0001, "step": 8, "time_profile/curr_logprobs_and_update": 0.2615692205145024, "train/gen_step": 512.0, "und/clip_ratio/high_mean": 0.00020409237549756654, "und/clip_ratio/low_mean": 0.00016332855375367217, "und/clip_ratio/region_mean": 0.0003674209328892175, "und/kl": 0.0024647556260788406, "und/loss": -4.9581576604396105e-05 }, { "epoch": 0.11529223378702963, "grad_norm": 0.05086541920900345, "learning_rate": 1e-05, "loss": 0.0001, "rewards/correctness_reward_func/mean": 0.6875, "rewards/correctness_reward_func/std": 0.9574271440505981, "rewards/strict_format_reward_func/mean": 0.2734375, "rewards/strict_format_reward_func/std": 0.250866562128067, "step": 9, "time_profile/curr_logprobs_and_update": 0.2616978839650983, "time_profile/old_logps": 33.66389245353639, "time_profile/ref_logps": 33.785877684131265, "time_profile/reward": 0.004252792336046696, "time_profile/text_rollout": 488.8415644085035, "train/gen_step": 576.0, "und/advantages_abs_mean": 0.7744140625, "und/advantages_max": 2.0625, "und/advantages_mean": 0.0, "und/advantages_min": -2.0625, "und/advantages_std": 0.9534159302711487, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.6171875, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.0, "und/kl": 0.0022935827773835626, "und/loss": 0.00010531611042097211, "und/prompt/max_length": 156.0, "und/prompt/mean_length": 156.0, "und/prompt/min_length": 156.0, "und/reward": 1.041015625, "und/reward_std": 1.1144300699234009 }, { "epoch": 0.12810248198558846, "grad_norm": 0.07067691534757614, "learning_rate": 1e-05, "loss": 0.0002, "step": 10, "time_profile/curr_logprobs_and_update": 0.26148990585352294, "train/gen_step": 640.0, "und/clip_ratio/high_mean": 0.000207764285732992, "und/clip_ratio/low_mean": 0.00026153815633733757, "und/clip_ratio/region_mean": 0.00046930244570830837, "und/kl": 0.0025965598733819206, "und/loss": -0.00032083288533613086 }, { "epoch": 0.14091273018414732, "grad_norm": 0.011798444204032421, "learning_rate": 1e-05, "loss": 0.0001, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.3359375, "rewards/strict_format_reward_func/std": 0.2366211861371994, "step": 11, "time_profile/curr_logprobs_and_update": 0.2613880275603151, "time_profile/old_logps": 33.668298315256834, "time_profile/ref_logps": 33.73314023669809, "time_profile/reward": 0.0044129379093647, "time_profile/text_rollout": 454.3680015280843, "train/gen_step": 704.0, "und/advantages_abs_mean": 0.1396484375, "und/advantages_max": 1.75, "und/advantages_mean": 0.0, "und/advantages_min": -0.4375, "und/advantages_std": 0.20029333233833313, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.6640625, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.28125, "und/kl": 0.0030043681945244316, "und/loss": 4.8866542471159846e-05, "und/prompt/max_length": 49.0, "und/prompt/mean_length": 49.0, "und/prompt/min_length": 49.0, "und/reward": 0.34765625, "und/reward_std": 0.25070229172706604 }, { "epoch": 0.15372297838270615, "grad_norm": 0.00986644346266985, "learning_rate": 1e-05, "loss": 0.0002, "step": 12, "time_profile/curr_logprobs_and_update": 0.2613714834296843, "train/gen_step": 768.0, "und/clip_ratio/high_mean": 0.0003061741153942421, "und/clip_ratio/low_mean": 0.0004524241703620646, "und/clip_ratio/region_mean": 0.0007585982857563067, "und/kl": 0.0042751526357278635, "und/loss": 0.0005581114168933254 }, { "epoch": 0.16653322658126501, "grad_norm": 0.044079430401325226, "learning_rate": 1e-05, "loss": 0.0002, "rewards/correctness_reward_func/mean": 0.78125, "rewards/correctness_reward_func/std": 0.983494758605957, "rewards/strict_format_reward_func/mean": 0.375, "rewards/strict_format_reward_func/std": 0.2182178944349289, "step": 13, "time_profile/curr_logprobs_and_update": 0.2613269358407706, "time_profile/old_logps": 33.61356293410063, "time_profile/ref_logps": 33.69876565504819, "time_profile/reward": 0.004282657988369465, "time_profile/text_rollout": 455.1681863386184, "train/gen_step": 832.0, "und/advantages_abs_mean": 0.33984375, "und/advantages_max": 1.0625, "und/advantages_mean": 0.0, "und/advantages_min": -2.1875, "und/advantages_std": 0.5323442816734314, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.69140625, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.171875, "und/kl": 0.00535378480435611, "und/loss": 0.00024319114163517952, "und/prompt/max_length": 55.0, "und/prompt/mean_length": 51.0, "und/prompt/min_length": 47.0, "und/reward": 1.21875, "und/reward_std": 1.1256657838821411 }, { "epoch": 0.17934347477982385, "grad_norm": 0.04768079146742821, "learning_rate": 1e-05, "loss": 0.0001, "step": 14, "time_profile/curr_logprobs_and_update": 0.2612948026799131, "train/gen_step": 896.0, "und/clip_ratio/high_mean": 0.0003154302030452527, "und/clip_ratio/low_mean": 0.0004365383902040776, "und/clip_ratio/region_mean": 0.0007519685932493303, "und/kl": 0.005003000936994795, "und/loss": -6.164377555251122e-05 }, { "epoch": 0.1921537229783827, "grad_norm": 0.013837647624313831, "learning_rate": 1e-05, "loss": 0.0001, "rewards/correctness_reward_func/mean": 0.03125, "rewards/correctness_reward_func/std": 0.25, "rewards/strict_format_reward_func/mean": 0.3203125, "rewards/strict_format_reward_func/std": 0.24180518090724945, "step": 15, "time_profile/curr_logprobs_and_update": 0.261544900276931, "time_profile/old_logps": 33.64624940324575, "time_profile/ref_logps": 33.727370373904705, "time_profile/reward": 0.004188773222267628, "time_profile/text_rollout": 454.6840885374695, "train/gen_step": 960.0, "und/advantages_abs_mean": 0.149658203125, "und/advantages_max": 1.9375, "und/advantages_mean": 0.0, "und/advantages_min": -0.5625, "und/advantages_std": 0.2218771129846573, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.619140625, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.203125, "und/kl": 0.002458287915942492, "und/loss": 0.00015533823614077846, "und/prompt/max_length": 51.0, "und/prompt/mean_length": 49.5, "und/prompt/min_length": 48.0, "und/reward": 0.3330078125, "und/reward_std": 0.2671593129634857 }, { "epoch": 0.20496397117694154, "grad_norm": 0.012836061418056488, "learning_rate": 1e-05, "loss": 0.0001, "step": 16, "time_profile/curr_logprobs_and_update": 0.2614677896053763, "train/gen_step": 1024.0, "und/clip_ratio/high_mean": 0.0010820379393408075, "und/clip_ratio/low_mean": 5.425347262644209e-05, "und/clip_ratio/region_mean": 0.0011362914119672496, "und/kl": 0.0026947305022986257, "und/loss": 1.8947941015312608e-05 }, { "epoch": 0.2177742193755004, "grad_norm": 0.03666432574391365, "learning_rate": 1e-05, "loss": 0.0002, "rewards/correctness_reward_func/mean": 0.125, "rewards/correctness_reward_func/std": 0.48795005679130554, "rewards/strict_format_reward_func/mean": 0.484375, "rewards/strict_format_reward_func/std": 0.08768405020236969, "step": 17, "time_profile/curr_logprobs_and_update": 0.26129884374677204, "time_profile/old_logps": 33.58372610062361, "time_profile/ref_logps": 33.71312084700912, "time_profile/reward": 0.004377733916044235, "time_profile/text_rollout": 453.4618791854009, "train/gen_step": 1088.0, "und/advantages_abs_mean": 0.24462890625, "und/advantages_max": 1.875, "und/advantages_mean": 0.0, "und/advantages_min": -0.9375, "und/advantages_std": 0.4919235110282898, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.58984375, "und/completion/min_length": 216.0, "und/frac_reward_zero_std": 0.453125, "und/kl": 0.003914871423148725, "und/loss": 0.00013992213200708647, "und/prompt/max_length": 45.0, "und/prompt/mean_length": 44.0, "und/prompt/min_length": 43.0, "und/reward": 0.6201171875, "und/reward_std": 0.5268946886062622 }, { "epoch": 0.23058446757405926, "grad_norm": 0.04468945786356926, "learning_rate": 1e-05, "loss": 0.0003, "step": 18, "time_profile/curr_logprobs_and_update": 0.26125960882927757, "train/gen_step": 1152.0, "und/clip_ratio/high_mean": 0.00032832663418957964, "und/clip_ratio/low_mean": 0.0001570866115798708, "und/clip_ratio/region_mean": 0.00048541324576945044, "und/kl": 0.004693901622886187, "und/loss": 0.0001238020138245588 }, { "epoch": 0.2433947157726181, "grad_norm": 0.04278712719678879, "learning_rate": 1e-05, "loss": 0.0002, "rewards/correctness_reward_func/mean": 0.34375, "rewards/correctness_reward_func/std": 0.7605084180831909, "rewards/strict_format_reward_func/mean": 0.4140625, "rewards/strict_format_reward_func/std": 0.19012710452079773, "step": 19, "time_profile/curr_logprobs_and_update": 0.2615380200877553, "time_profile/old_logps": 33.66465362161398, "time_profile/ref_logps": 33.743118970654905, "time_profile/reward": 0.00426104012876749, "time_profile/text_rollout": 454.9674591496587, "train/gen_step": 1216.0, "und/advantages_abs_mean": 0.5419921875, "und/advantages_max": 1.875, "und/advantages_mean": 0.0, "und/advantages_min": -1.6875, "und/advantages_std": 0.6924689412117004, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.603515625, "und/completion/min_length": 247.0, "und/frac_reward_zero_std": 0.0, "und/kl": 0.005255785068584373, "und/loss": 0.000339222839102149, "und/prompt/max_length": 53.0, "und/prompt/mean_length": 52.0, "und/prompt/min_length": 51.0, "und/reward": 0.7529296875, "und/reward_std": 0.8764092326164246 }, { "epoch": 0.2562049639711769, "grad_norm": 0.05156658962368965, "learning_rate": 1e-05, "loss": 0.0005, "step": 20, "time_profile/curr_logprobs_and_update": 0.2612772498978302, "train/gen_step": 1280.0, "und/clip_ratio/high_mean": 0.000682802070514299, "und/clip_ratio/low_mean": 0.0004532934799499344, "und/clip_ratio/region_mean": 0.0011360955541022122, "und/kl": 0.007088688389558229, "und/loss": -0.000385153922252357 }, { "epoch": 0.2690152121697358, "grad_norm": 0.010599356144666672, "learning_rate": 1e-05, "loss": 0.0004, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.03125, "rewards/strict_format_reward_func/std": 0.12198751419782639, "step": 21, "time_profile/curr_logprobs_and_update": 0.26165466259408277, "time_profile/old_logps": 33.66928117442876, "time_profile/ref_logps": 33.74430500715971, "time_profile/reward": 0.005226008594036102, "time_profile/text_rollout": 505.46357171423733, "train/gen_step": 1344.0, "und/advantages_abs_mean": 0.0986328125, "und/advantages_max": 2.1875, "und/advantages_mean": 0.0, "und/advantages_min": -0.375, "und/advantages_std": 0.23039571940898895, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.90625, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.546875, "und/kl": 0.009255951059458312, "und/loss": 0.0003061078954686991, "und/prompt/max_length": 247.0, "und/prompt/mean_length": 204.5, "und/prompt/min_length": 162.0, "und/reward": 0.068359375, "und/reward_std": 0.2506641745567322 }, { "epoch": 0.28182546036829464, "grad_norm": 0.006633961573243141, "learning_rate": 1e-05, "loss": 0.0005, "step": 22, "time_profile/curr_logprobs_and_update": 0.2616323969559744, "train/gen_step": 1408.0, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.00017277644656132907, "und/clip_ratio/region_mean": 0.00017277644656132907, "und/kl": 0.01153348765183182, "und/loss": 0.0003008711780587703 }, { "epoch": 0.2946357085668535, "grad_norm": 0.01536457147449255, "learning_rate": 1e-05, "loss": 0.0005, "rewards/correctness_reward_func/mean": 0.90625, "rewards/correctness_reward_func/std": 1.003466248512268, "rewards/strict_format_reward_func/mean": 0.34375, "rewards/strict_format_reward_func/std": 0.233588308095932, "step": 23, "time_profile/curr_logprobs_and_update": 0.26142213236016687, "time_profile/old_logps": 33.64063487108797, "time_profile/ref_logps": 33.71632799040526, "time_profile/reward": 0.004507049918174744, "time_profile/text_rollout": 456.5940729761496, "train/gen_step": 1472.0, "und/advantages_abs_mean": 0.3193359375, "und/advantages_max": 1.0, "und/advantages_mean": 0.0, "und/advantages_min": -2.1875, "und/advantages_std": 0.508188784122467, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.421875, "und/completion/min_length": 126.0, "und/frac_reward_zero_std": 0.109375, "und/kl": 0.011904407194379019, "und/loss": 0.000405974641296325, "und/prompt/max_length": 68.0, "und/prompt/mean_length": 57.0, "und/prompt/min_length": 46.0, "und/reward": 1.1806640625, "und/reward_std": 1.1607900857925415 }, { "epoch": 0.3074459567654123, "grad_norm": 0.017077524214982986, "learning_rate": 1e-05, "loss": 0.0007, "step": 24, "time_profile/curr_logprobs_and_update": 0.26139352870814037, "train/gen_step": 1536.0, "und/clip_ratio/high_mean": 0.0003636985675257165, "und/clip_ratio/low_mean": 0.000808012075140141, "und/clip_ratio/region_mean": 0.0011717106426658574, "und/kl": 0.0162865712918574, "und/loss": 0.0004572992978353341 }, { "epoch": 0.32025620496397117, "grad_norm": 0.011341015808284283, "learning_rate": 1e-05, "loss": 0.0009, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.15625, "rewards/strict_format_reward_func/std": 0.233588308095932, "step": 25, "time_profile/curr_logprobs_and_update": 0.2614934611192439, "time_profile/old_logps": 33.66488476470113, "time_profile/ref_logps": 33.74662786722183, "time_profile/reward": 0.0043584490194916725, "time_profile/text_rollout": 473.53795502893627, "train/gen_step": 1600.0, "und/advantages_abs_mean": 0.18212890625, "und/advantages_max": 2.1875, "und/advantages_mean": 0.0, "und/advantages_min": -0.3125, "und/advantages_std": 0.2327723652124405, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.763671875, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.078125, "und/kl": 0.02267802683309128, "und/loss": 0.001361295189667544, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 109.5, "und/prompt/min_length": 57.0, "und/reward": 0.14453125, "und/reward_std": 0.2475108504295349 }, { "epoch": 0.33306645316253003, "grad_norm": 0.014138264581561089, "learning_rate": 1e-05, "loss": 0.0014, "step": 26, "time_profile/curr_logprobs_and_update": 0.2614813649561256, "train/gen_step": 1664.0, "und/clip_ratio/high_mean": 9.385726662003435e-05, "und/clip_ratio/low_mean": 0.0006062783650122583, "und/clip_ratio/region_mean": 0.0007001356316322926, "und/kl": 0.03298538580929744, "und/loss": 0.0003666753910565035 }, { "epoch": 0.3458767013610889, "grad_norm": 0.036602944135665894, "learning_rate": 1e-05, "loss": 0.0017, "rewards/correctness_reward_func/mean": 0.15625, "rewards/correctness_reward_func/std": 0.5409794449806213, "rewards/strict_format_reward_func/mean": 0.296875, "rewards/strict_format_reward_func/std": 0.24750742316246033, "step": 27, "time_profile/curr_logprobs_and_update": 0.26141884991375264, "time_profile/old_logps": 33.63414192106575, "time_profile/ref_logps": 33.71741700544953, "time_profile/reward": 0.004512332379817963, "time_profile/text_rollout": 454.9066513767466, "train/gen_step": 1728.0, "und/advantages_abs_mean": 0.361572265625, "und/advantages_max": 2.0, "und/advantages_mean": 0.0, "und/advantages_min": -1.25, "und/advantages_std": 0.5393062829971313, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.65234375, "und/completion/min_length": 250.0, "und/frac_reward_zero_std": 0.015625, "und/kl": 0.04265276483056368, "und/loss": 0.0006861621513962746, "und/prompt/max_length": 59.0, "und/prompt/mean_length": 52.0, "und/prompt/min_length": 45.0, "und/reward": 0.44921875, "und/reward_std": 0.6556687951087952 }, { "epoch": 0.3586869495596477, "grad_norm": 0.05098080635070801, "learning_rate": 1e-05, "loss": 0.0015, "step": 28, "time_profile/curr_logprobs_and_update": 0.261381899996195, "train/gen_step": 1792.0, "und/clip_ratio/high_mean": 0.0009093214357562829, "und/clip_ratio/low_mean": 0.0001023065487970598, "und/clip_ratio/region_mean": 0.0010116279845533427, "und/kl": 0.037396987856482156, "und/loss": 0.0021928895730525255 }, { "epoch": 0.37149719775820655, "grad_norm": 0.009760422632098198, "learning_rate": 1e-05, "loss": 0.0023, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.2734375, "rewards/strict_format_reward_func/std": 0.250866562128067, "step": 29, "time_profile/curr_logprobs_and_update": 0.26138593546056654, "time_profile/old_logps": 33.63254508841783, "time_profile/ref_logps": 33.774293441325426, "time_profile/reward": 0.004322300665080547, "time_profile/text_rollout": 455.36977090034634, "train/gen_step": 1856.0, "und/advantages_abs_mean": 0.2197265625, "und/advantages_max": 0.4375, "und/advantages_mean": 0.0, "und/advantages_min": -0.4375, "und/advantages_std": 0.23460420966148376, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.712890625, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.015625, "und/kl": 0.056478848066035425, "und/loss": 0.006321181455859914, "und/prompt/max_length": 54.0, "und/prompt/mean_length": 53.5, "und/prompt/min_length": 53.0, "und/reward": 0.236328125, "und/reward_std": 0.24987001717090607 }, { "epoch": 0.3843074459567654, "grad_norm": 0.010228869505226612, "learning_rate": 1e-05, "loss": 0.0015, "step": 30, "time_profile/curr_logprobs_and_update": 0.2613552274706308, "train/gen_step": 1920.0, "und/clip_ratio/high_mean": 0.0011887354085047264, "und/clip_ratio/low_mean": 4.245923992129974e-05, "und/clip_ratio/region_mean": 0.001231194648426026, "und/kl": 0.023746514114463935, "und/loss": 0.0005230403039604425 }, { "epoch": 0.3971176941553243, "grad_norm": 0.050902411341667175, "learning_rate": 1e-05, "loss": 0.0008, "rewards/correctness_reward_func/mean": 0.5, "rewards/correctness_reward_func/std": 0.8728715777397156, "rewards/strict_format_reward_func/mean": 0.34375, "rewards/strict_format_reward_func/std": 0.233588308095932, "step": 31, "time_profile/curr_logprobs_and_update": 0.26159091785666533, "time_profile/old_logps": 33.64319808781147, "time_profile/ref_logps": 33.71481241378933, "time_profile/reward": 0.004479923285543919, "time_profile/text_rollout": 473.2088502328843, "train/gen_step": 1984.0, "und/advantages_abs_mean": 0.535400390625, "und/advantages_max": 1.5625, "und/advantages_mean": 0.0, "und/advantages_min": -1.75, "und/advantages_std": 0.6867490410804749, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.625, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.015625, "und/kl": 0.018857156013837084, "und/loss": 0.0008609343785792589, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 108.0, "und/prompt/min_length": 54.0, "und/reward": 0.83203125, "und/reward_std": 1.0073648691177368 }, { "epoch": 0.4099279423538831, "grad_norm": 0.06242002546787262, "learning_rate": 1e-05, "loss": 0.0015, "step": 32, "time_profile/curr_logprobs_and_update": 0.26177428937808145, "train/gen_step": 2048.0, "und/clip_ratio/high_mean": 0.0007951511470309924, "und/clip_ratio/low_mean": 0.00020771568233612925, "und/clip_ratio/region_mean": 0.0010028668257291429, "und/kl": 0.03375891799441888, "und/loss": 0.000749626662582159 }, { "epoch": 0.42273819055244194, "grad_norm": 0.04212071746587753, "learning_rate": 1e-05, "loss": 0.0006, "rewards/correctness_reward_func/mean": 0.5, "rewards/correctness_reward_func/std": 0.8728715777397156, "rewards/strict_format_reward_func/mean": 0.3125, "rewards/strict_format_reward_func/std": 0.24397502839565277, "step": 33, "time_profile/curr_logprobs_and_update": 0.26152711105532944, "time_profile/old_logps": 33.65837723109871, "time_profile/ref_logps": 33.75508733931929, "time_profile/reward": 0.0049184514209628105, "time_profile/text_rollout": 472.5054632341489, "train/gen_step": 2112.0, "und/advantages_abs_mean": 0.53515625, "und/advantages_max": 1.75, "und/advantages_mean": 0.0, "und/advantages_min": -1.75, "und/advantages_std": 0.6916294097900391, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.671875, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.015625, "und/kl": 0.015675050650315825, "und/loss": 0.0005543280858546495, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 104.5, "und/prompt/min_length": 47.0, "und/reward": 0.84375, "und/reward_std": 1.0150530338287354 }, { "epoch": 0.4355484387510008, "grad_norm": 0.0346144400537014, "learning_rate": 1e-05, "loss": 0.0012, "step": 34, "time_profile/curr_logprobs_and_update": 0.2614700558333425, "train/gen_step": 2176.0, "und/clip_ratio/high_mean": 0.0002005133756028954, "und/clip_ratio/low_mean": 0.00024482345179421827, "und/clip_ratio/region_mean": 0.00044533682739711367, "und/kl": 0.025506131180009106, "und/loss": 0.001075362495612353 }, { "epoch": 0.44835868694955966, "grad_norm": 0.04864125698804855, "learning_rate": 1e-05, "loss": 0.0008, "rewards/correctness_reward_func/mean": 0.1875, "rewards/correctness_reward_func/std": 0.5875696539878845, "rewards/strict_format_reward_func/mean": 0.25, "rewards/strict_format_reward_func/std": 0.2519763112068176, "step": 35, "time_profile/curr_logprobs_and_update": 0.2617200283857528, "time_profile/old_logps": 33.63409499172121, "time_profile/ref_logps": 33.74282674957067, "time_profile/reward": 0.004408497363328934, "time_profile/text_rollout": 487.6969060283154, "train/gen_step": 2240.0, "und/advantages_abs_mean": 0.337890625, "und/advantages_max": 1.875, "und/advantages_mean": 0.0, "und/advantages_min": -1.4375, "und/advantages_std": 0.585414707660675, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.32421875, "und/completion/min_length": 175.0, "und/frac_reward_zero_std": 0.46875, "und/kl": 0.019317339225381147, "und/loss": 0.0008824675582559394, "und/prompt/max_length": 247.0, "und/prompt/mean_length": 145.0, "und/prompt/min_length": 43.0, "und/reward": 0.478515625, "und/reward_std": 0.7729067802429199 }, { "epoch": 0.4611689351481185, "grad_norm": 0.043183691799640656, "learning_rate": 1e-05, "loss": 0.0009, "step": 36, "time_profile/curr_logprobs_and_update": 0.26155611700960435, "train/gen_step": 2304.0, "und/clip_ratio/high_mean": 0.00021402155107352883, "und/clip_ratio/low_mean": 0.00021690024368581362, "und/clip_ratio/region_mean": 0.00043092179839732125, "und/kl": 0.024045594029303174, "und/loss": 0.0011579781168506997 }, { "epoch": 0.4739791833466773, "grad_norm": 0.010638823732733727, "learning_rate": 1e-05, "loss": 0.0009, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.109375, "rewards/strict_format_reward_func/std": 0.2083333432674408, "step": 37, "time_profile/curr_logprobs_and_update": 0.2616240307397675, "time_profile/old_logps": 33.59795498009771, "time_profile/ref_logps": 33.70516960695386, "time_profile/reward": 0.004617785103619099, "time_profile/text_rollout": 487.4186108149588, "train/gen_step": 2368.0, "und/advantages_abs_mean": 0.136962890625, "und/advantages_max": 2.1875, "und/advantages_mean": 0.0, "und/advantages_min": -0.625, "und/advantages_std": 0.25544461607933044, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.880859375, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.4375, "und/kl": 0.022708220010827063, "und/loss": 0.0009557972493325906, "und/prompt/max_length": 247.0, "und/prompt/mean_length": 151.0, "und/prompt/min_length": 55.0, "und/reward": 0.1318359375, "und/reward_std": 0.2962620258331299 }, { "epoch": 0.4867894315452362, "grad_norm": 0.010083939880132675, "learning_rate": 1e-05, "loss": 0.001, "step": 38, "time_profile/curr_logprobs_and_update": 0.2614540692156879, "train/gen_step": 2432.0, "und/clip_ratio/high_mean": 0.000718017738108756, "und/clip_ratio/low_mean": 0.00015644555605831556, "und/clip_ratio/region_mean": 0.0008744632941670716, "und/kl": 0.024760687578236684, "und/loss": 0.001179238304757746 }, { "epoch": 0.49959967974379504, "grad_norm": 0.03500046208500862, "learning_rate": 1e-05, "loss": 0.0015, "rewards/correctness_reward_func/mean": 0.71875, "rewards/correctness_reward_func/std": 0.9672207236289978, "rewards/strict_format_reward_func/mean": 0.234375, "rewards/strict_format_reward_func/std": 0.2514837086200714, "step": 39, "time_profile/curr_logprobs_and_update": 0.2616706858971156, "time_profile/old_logps": 33.69548157788813, "time_profile/ref_logps": 33.755788206122816, "time_profile/reward": 0.004642653279006481, "time_profile/text_rollout": 503.46805787831545, "train/gen_step": 2496.0, "und/advantages_abs_mean": 0.399658203125, "und/advantages_max": 1.375, "und/advantages_mean": 0.0, "und/advantages_min": -1.9375, "und/advantages_std": 0.6456708908081055, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 253.01953125, "und/completion/min_length": 21.0, "und/frac_reward_zero_std": 0.515625, "und/kl": 0.03765736366040073, "und/loss": 0.000907381055640144, "und/prompt/max_length": 313.0, "und/prompt/mean_length": 179.5, "und/prompt/min_length": 46.0, "und/reward": 0.9072265625, "und/reward_std": 1.1414990425109863 }, { "epoch": 0.5124099279423538, "grad_norm": 0.047211844474077225, "learning_rate": 1e-05, "loss": 0.0014, "step": 40, "time_profile/curr_logprobs_and_update": 0.2616095836274326, "train/gen_step": 2560.0, "und/clip_ratio/high_mean": 0.00029819617702742107, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.00029819617702742107, "und/kl": 0.03165427554631606, "und/loss": 0.0009896907330357863 }, { "epoch": 0.5252201761409128, "grad_norm": 0.012100051157176495, "learning_rate": 1e-05, "loss": 0.0132, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.171875, "rewards/strict_format_reward_func/std": 0.23935678601264954, "step": 41, "time_profile/curr_logprobs_and_update": 0.26168578446959145, "time_profile/old_logps": 33.68038745317608, "time_profile/ref_logps": 33.777247093617916, "time_profile/reward": 0.004466196522116661, "time_profile/text_rollout": 491.4084973083809, "train/gen_step": 2624.0, "und/advantages_abs_mean": 0.169189453125, "und/advantages_max": 0.4375, "und/advantages_mean": 0.0, "und/advantages_min": -0.375, "und/advantages_std": 0.20586435496807098, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.802734375, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.109375, "und/kl": 0.33045044860773487, "und/loss": 0.013275288394652307, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 162.0, "und/prompt/min_length": 162.0, "und/reward": 0.1376953125, "und/reward_std": 0.22357389330863953 }, { "epoch": 0.5380304243394716, "grad_norm": 0.009308804757893085, "learning_rate": 1e-05, "loss": 0.0254, "step": 42, "time_profile/curr_logprobs_and_update": 0.26165859241154976, "train/gen_step": 2688.0, "und/clip_ratio/high_mean": 0.00019616381541709416, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.00019616381541709416, "und/kl": 0.6357102408874198, "und/loss": 0.028003237792290747 }, { "epoch": 0.5508406725380304, "grad_norm": 0.055917106568813324, "learning_rate": 1e-05, "loss": 0.0037, "rewards/correctness_reward_func/mean": 0.5625, "rewards/correctness_reward_func/std": 0.9063270092010498, "rewards/strict_format_reward_func/mean": 0.3125, "rewards/strict_format_reward_func/std": 0.24397502839565277, "step": 43, "time_profile/curr_logprobs_and_update": 0.26144762606418226, "time_profile/old_logps": 33.624562768265605, "time_profile/ref_logps": 33.73785715736449, "time_profile/reward": 0.004686887376010418, "time_profile/text_rollout": 454.951701448299, "train/gen_step": 2752.0, "und/advantages_abs_mean": 0.566650390625, "und/advantages_max": 1.5, "und/advantages_mean": 0.0, "und/advantages_min": -1.9375, "und/advantages_std": 0.7055483460426331, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.75390625, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.0, "und/kl": 0.09146726290055085, "und/loss": 0.0016110616270452738, "und/prompt/max_length": 57.0, "und/prompt/mean_length": 50.5, "und/prompt/min_length": 44.0, "und/reward": 0.859375, "und/reward_std": 0.9939887523651123 }, { "epoch": 0.5636509207365893, "grad_norm": 0.0613945871591568, "learning_rate": 1e-05, "loss": 0.0074, "step": 44, "time_profile/curr_logprobs_and_update": 0.26146509549289476, "train/gen_step": 2816.0, "und/clip_ratio/high_mean": 0.00215041778574232, "und/clip_ratio/low_mean": 0.00034797564512700774, "und/clip_ratio/region_mean": 0.002498393430869328, "und/kl": 0.0361222850260674, "und/loss": 0.005510351620614529 }, { "epoch": 0.5764611689351481, "grad_norm": 0.009947679936885834, "learning_rate": 1e-05, "loss": 0.0012, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.28125, "rewards/strict_format_reward_func/std": 0.25, "step": 45, "time_profile/curr_logprobs_and_update": 0.26160548186453525, "time_profile/old_logps": 33.671746659092605, "time_profile/ref_logps": 33.75626815017313, "time_profile/reward": 0.3108100714161992, "time_profile/text_rollout": 472.72174780536443, "train/gen_step": 2880.0, "und/advantages_abs_mean": 0.1962890625, "und/advantages_max": 0.4375, "und/advantages_mean": 0.0, "und/advantages_min": -0.4375, "und/advantages_std": 0.2217392474412918, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.67578125, "und/completion/min_length": 250.0, "und/frac_reward_zero_std": 0.046875, "und/kl": 0.02999453744996572, "und/loss": 0.0007559580495524187, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 104.5, "und/prompt/min_length": 47.0, "und/reward": 0.255859375, "und/reward_std": 0.25017574429512024 }, { "epoch": 0.589271417133707, "grad_norm": 0.012918645516037941, "learning_rate": 1e-05, "loss": 0.0016, "step": 46, "time_profile/curr_logprobs_and_update": 0.2616680209466722, "train/gen_step": 2944.0, "und/clip_ratio/high_mean": 0.0007937824193504639, "und/clip_ratio/low_mean": 4.763719334732741e-05, "und/clip_ratio/region_mean": 0.0008414196126977913, "und/kl": 0.03639418514649151, "und/loss": 0.001037593943193471 }, { "epoch": 0.6020816653322658, "grad_norm": 0.005492714699357748, "learning_rate": 1e-05, "loss": 0.0545, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.140625, "rewards/strict_format_reward_func/std": 0.22658175230026245, "step": 47, "time_profile/curr_logprobs_and_update": 0.2618762481288286, "time_profile/old_logps": 33.7198549490422, "time_profile/ref_logps": 33.79357417766005, "time_profile/reward": 0.004348478280007839, "time_profile/text_rollout": 491.14609040971845, "train/gen_step": 3008.0, "und/advantages_abs_mean": 0.188232421875, "und/advantages_max": 1.9375, "und/advantages_mean": 0.0, "und/advantages_min": -0.5625, "und/advantages_std": 0.24418380856513977, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.794921875, "und/completion/min_length": 252.0, "und/frac_reward_zero_std": 0.046875, "und/kl": 1.3629103746789042, "und/loss": 0.0005332264117896557, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 162.0, "und/prompt/min_length": 162.0, "und/reward": 0.1552734375, "und/reward_std": 0.26322463154792786 }, { "epoch": 0.6148919135308246, "grad_norm": 0.007346948143094778, "learning_rate": 1e-05, "loss": 0.0133, "step": 48, "time_profile/curr_logprobs_and_update": 0.26197397308715153, "train/gen_step": 3072.0, "und/clip_ratio/high_mean": 4.882812572759576e-05, "und/clip_ratio/low_mean": 7.812499825377017e-05, "und/clip_ratio/region_mean": 0.00012695312398136593, "und/kl": 0.3335349300832604, "und/loss": 0.000871355994604528 }, { "epoch": 0.6277021617293835, "grad_norm": 0.041968513280153275, "learning_rate": 1e-05, "loss": 0.0017, "rewards/correctness_reward_func/mean": 0.71875, "rewards/correctness_reward_func/std": 0.9672207236289978, "rewards/strict_format_reward_func/mean": 0.3359375, "rewards/strict_format_reward_func/std": 0.2366211861371994, "step": 49, "time_profile/curr_logprobs_and_update": 0.26143342345312703, "time_profile/old_logps": 33.66150098387152, "time_profile/ref_logps": 33.79388515371829, "time_profile/reward": 0.0042456211522221565, "time_profile/text_rollout": 455.7687773555517, "train/gen_step": 3136.0, "und/advantages_abs_mean": 0.4990234375, "und/advantages_max": 1.5625, "und/advantages_mean": 0.0, "und/advantages_min": -2.1875, "und/advantages_std": 0.6655394434928894, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.501953125, "und/completion/min_length": 159.0, "und/frac_reward_zero_std": 0.03125, "und/kl": 0.043663310549163725, "und/loss": 0.001001961762085557, "und/prompt/max_length": 57.0, "und/prompt/mean_length": 52.5, "und/prompt/min_length": 48.0, "und/reward": 0.9833984375, "und/reward_std": 1.0895419120788574 }, { "epoch": 0.6405124099279423, "grad_norm": 0.03877226635813713, "learning_rate": 1e-05, "loss": 0.0014, "step": 50, "time_profile/curr_logprobs_and_update": 0.26151250462862663, "train/gen_step": 3200.0, "und/clip_ratio/high_mean": 0.0006906339149281848, "und/clip_ratio/low_mean": 0.001115720056986902, "und/clip_ratio/region_mean": 0.0018063539719150867, "und/kl": 0.03958344936108915, "und/loss": 0.0018563272897154093 }, { "epoch": 0.6533226581265013, "grad_norm": 0.040287211537361145, "learning_rate": 1e-05, "loss": 0.0016, "rewards/correctness_reward_func/mean": 0.8125, "rewards/correctness_reward_func/std": 0.9900296926498413, "rewards/strict_format_reward_func/mean": 0.359375, "rewards/strict_format_reward_func/std": 0.22658175230026245, "step": 51, "time_profile/curr_logprobs_and_update": 0.2614360564039089, "time_profile/old_logps": 33.677048296667635, "time_profile/ref_logps": 33.734667072072625, "time_profile/reward": 0.00466107577085495, "time_profile/text_rollout": 454.67956179752946, "train/gen_step": 3264.0, "und/advantages_abs_mean": 0.498779296875, "und/advantages_max": 1.8125, "und/advantages_mean": 0.0, "und/advantages_min": -2.1875, "und/advantages_std": 0.669204831123352, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.6875, "und/completion/min_length": 251.0, "und/frac_reward_zero_std": 0.015625, "und/kl": 0.03883673060772708, "und/loss": 0.001454857557376954, "und/prompt/max_length": 55.0, "und/prompt/mean_length": 52.0, "und/prompt/min_length": 49.0, "und/reward": 0.9443359375, "und/reward_std": 1.075581669807434 }, { "epoch": 0.6661329063250601, "grad_norm": 0.038634829223155975, "learning_rate": 1e-05, "loss": 0.0012, "step": 52, "time_profile/curr_logprobs_and_update": 0.2613996758009307, "train/gen_step": 3328.0, "und/clip_ratio/high_mean": 0.0003874025678669568, "und/clip_ratio/low_mean": 0.00019797183631453663, "und/clip_ratio/region_mean": 0.0005853744041814934, "und/kl": 0.03153260219551157, "und/loss": 0.0016808376690278237 }, { "epoch": 0.6789431545236189, "grad_norm": 0.05004860833287239, "learning_rate": 1e-05, "loss": 0.0109, "rewards/correctness_reward_func/mean": 0.15625, "rewards/correctness_reward_func/std": 0.5409794449806213, "rewards/strict_format_reward_func/mean": 0.1953125, "rewards/strict_format_reward_func/std": 0.24587368965148926, "step": 53, "time_profile/curr_logprobs_and_update": 0.26163243876362685, "time_profile/old_logps": 33.65926602482796, "time_profile/ref_logps": 33.75773391965777, "time_profile/reward": 0.004499710164964199, "time_profile/text_rollout": 516.6223611980677, "train/gen_step": 3392.0, "und/advantages_abs_mean": 0.2509765625, "und/advantages_max": 2.0625, "und/advantages_mean": 0.0, "und/advantages_min": -1.125, "und/advantages_std": 0.5241244435310364, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.7734375, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.5, "und/kl": 0.2721175526821753, "und/loss": 0.054259198752447446, "und/prompt/max_length": 313.0, "und/prompt/mean_length": 234.5, "und/prompt/min_length": 156.0, "und/reward": 0.322265625, "und/reward_std": 0.6323413252830505 }, { "epoch": 0.6917534027221778, "grad_norm": 0.035311710089445114, "learning_rate": 1e-05, "loss": 0.0031, "step": 54, "time_profile/curr_logprobs_and_update": 0.26147235184907913, "train/gen_step": 3456.0, "und/clip_ratio/high_mean": 0.00019807044009212404, "und/clip_ratio/low_mean": 9.089218292501755e-05, "und/clip_ratio/region_mean": 0.0002889626230171416, "und/kl": 0.0721228092443198, "und/loss": 0.002958318164701268 }, { "epoch": 0.7045636509207366, "grad_norm": 0.04633024334907532, "learning_rate": 1e-05, "loss": 0.0015, "rewards/correctness_reward_func/mean": 0.90625, "rewards/correctness_reward_func/std": 1.003466248512268, "rewards/strict_format_reward_func/mean": 0.484375, "rewards/strict_format_reward_func/std": 0.08768405020236969, "step": 55, "time_profile/curr_logprobs_and_update": 0.2614114769385196, "time_profile/old_logps": 33.599201523698866, "time_profile/ref_logps": 33.73441112972796, "time_profile/reward": 0.004696068353950977, "time_profile/text_rollout": 453.6825319575146, "train/gen_step": 3520.0, "und/advantages_abs_mean": 0.341064453125, "und/advantages_max": 1.875, "und/advantages_mean": 0.0, "und/advantages_min": -2.1875, "und/advantages_std": 0.6085155010223389, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.72265625, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.40625, "und/kl": 0.03839961695484817, "und/loss": 0.0013346272653507185, "und/prompt/max_length": 47.0, "und/prompt/mean_length": 45.5, "und/prompt/min_length": 44.0, "und/reward": 1.3662109375, "und/reward_std": 1.0347063541412354 }, { "epoch": 0.7173738991192954, "grad_norm": 0.047499582171440125, "learning_rate": 1e-05, "loss": 0.0034, "step": 56, "time_profile/curr_logprobs_and_update": 0.2612421384692425, "train/gen_step": 3584.0, "und/clip_ratio/high_mean": 0.0003667974451673217, "und/clip_ratio/low_mean": 0.00013469014447764494, "und/clip_ratio/region_mean": 0.0005014875860069878, "und/kl": 0.08546131956973113, "und/loss": 0.00697715762282769 }, { "epoch": 0.7301841473178543, "grad_norm": 0.014783147722482681, "learning_rate": 1e-05, "loss": 0.004, "rewards/correctness_reward_func/mean": 0.09375, "rewards/correctness_reward_func/std": 0.42608407139778137, "rewards/strict_format_reward_func/mean": 0.296875, "rewards/strict_format_reward_func/std": 0.24750742316246033, "step": 57, "time_profile/curr_logprobs_and_update": 0.2614298419211991, "time_profile/old_logps": 33.64691087603569, "time_profile/ref_logps": 33.739184183999896, "time_profile/reward": 0.004393537528812885, "time_profile/text_rollout": 454.8285795925185, "train/gen_step": 3648.0, "und/advantages_abs_mean": 0.20654296875, "und/advantages_max": 1.8125, "und/advantages_mean": 0.0, "und/advantages_min": -0.9375, "und/advantages_std": 0.3522537350654602, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.716796875, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.21875, "und/kl": 0.10082034709921572, "und/loss": 0.0015968736425975294, "und/prompt/max_length": 56.0, "und/prompt/mean_length": 51.0, "und/prompt/min_length": 46.0, "und/reward": 0.3818359375, "und/reward_std": 0.42684176564216614 }, { "epoch": 0.7429943955164131, "grad_norm": 0.014119211584329605, "learning_rate": 1e-05, "loss": 0.0117, "step": 58, "time_profile/curr_logprobs_and_update": 0.26134903807542287, "train/gen_step": 3712.0, "und/clip_ratio/high_mean": 4.882812572759576e-05, "und/clip_ratio/low_mean": 0.00023804241573088802, "und/clip_ratio/region_mean": 0.0002868705414584838, "und/kl": 0.2931558708951343, "und/loss": 0.001128198406345149 }, { "epoch": 0.755804643714972, "grad_norm": 0.03397494554519653, "learning_rate": 1e-05, "loss": 0.0028, "rewards/correctness_reward_func/mean": 0.15625, "rewards/correctness_reward_func/std": 0.5409794449806213, "rewards/strict_format_reward_func/mean": 0.484375, "rewards/strict_format_reward_func/std": 0.08768405020236969, "step": 59, "time_profile/curr_logprobs_and_update": 0.2614648619637592, "time_profile/old_logps": 33.627349744550884, "time_profile/ref_logps": 33.70189340412617, "time_profile/reward": 0.00450173020362854, "time_profile/text_rollout": 454.01288072485477, "train/gen_step": 3776.0, "und/advantages_abs_mean": 0.373046875, "und/advantages_max": 1.8125, "und/advantages_mean": 0.0, "und/advantages_min": -1.6875, "und/advantages_std": 0.6158081293106079, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.609375, "und/completion/min_length": 251.0, "und/frac_reward_zero_std": 0.453125, "und/kl": 0.07084466355445329, "und/loss": 0.002370536963894665, "und/prompt/max_length": 47.0, "und/prompt/mean_length": 46.0, "und/prompt/min_length": 45.0, "und/reward": 0.7958984375, "und/reward_std": 0.739467978477478 }, { "epoch": 0.7686148919135308, "grad_norm": 0.046682216227054596, "learning_rate": 1e-05, "loss": 0.0041, "step": 60, "time_profile/curr_logprobs_and_update": 0.26129540299007203, "train/gen_step": 3840.0, "und/clip_ratio/high_mean": 0.000178795224201167, "und/clip_ratio/low_mean": 0.0001412087440257892, "und/clip_ratio/region_mean": 0.0003200039645889774, "und/kl": 0.10260857833782211, "und/loss": 0.0027817148465487662 }, { "epoch": 0.7814251401120896, "grad_norm": 0.014530722051858902, "learning_rate": 1e-05, "loss": 0.0184, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.1953125, "rewards/strict_format_reward_func/std": 0.24587368965148926, "step": 61, "time_profile/curr_logprobs_and_update": 0.2614762183802668, "time_profile/old_logps": 33.65403878968209, "time_profile/ref_logps": 33.731373984366655, "time_profile/reward": 0.004584863781929016, "time_profile/text_rollout": 472.9927922273055, "train/gen_step": 3904.0, "und/advantages_abs_mean": 0.203125, "und/advantages_max": 1.5, "und/advantages_mean": 0.0, "und/advantages_min": -0.5, "und/advantages_std": 0.23512499034404755, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.74609375, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.015625, "und/kl": 0.4595181476797734, "und/loss": 0.0014087316812947392, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 106.0, "und/prompt/min_length": 50.0, "und/reward": 0.20703125, "und/reward_std": 0.25815349817276 }, { "epoch": 0.7942353883106485, "grad_norm": 0.014059546403586864, "learning_rate": 1e-05, "loss": 0.0132, "step": 62, "time_profile/curr_logprobs_and_update": 0.2614812095707748, "train/gen_step": 3968.0, "und/clip_ratio/high_mean": 0.0003522909028106369, "und/clip_ratio/low_mean": 0.00013392856999416836, "und/clip_ratio/region_mean": 0.00048621947280480526, "und/kl": 0.32975731499755057, "und/loss": 0.0033024766016751528 }, { "epoch": 0.8070456365092074, "grad_norm": 0.013789220713078976, "learning_rate": 1e-05, "loss": 0.0013, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.2890625, "rewards/strict_format_reward_func/std": 0.24888142943382263, "step": 63, "time_profile/curr_logprobs_and_update": 0.26154088985640556, "time_profile/old_logps": 33.65780791267753, "time_profile/ref_logps": 33.74037241283804, "time_profile/reward": 0.003031497821211815, "time_profile/text_rollout": 454.97051773872226, "train/gen_step": 4032.0, "und/advantages_abs_mean": 0.210693359375, "und/advantages_max": 0.375, "und/advantages_mean": 0.0, "und/advantages_min": -0.4375, "und/advantages_std": 0.22973118722438812, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.74609375, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.046875, "und/kl": 0.033402759159798734, "und/loss": 0.0011463420676136593, "und/prompt/max_length": 52.0, "und/prompt/mean_length": 50.5, "und/prompt/min_length": 49.0, "und/reward": 0.2783203125, "und/reward_std": 0.24863366782665253 }, { "epoch": 0.8198558847077662, "grad_norm": 0.015820274129509926, "learning_rate": 1e-05, "loss": 0.0016, "step": 64, "time_profile/curr_logprobs_and_update": 0.2614853223931277, "train/gen_step": 4096.0, "und/clip_ratio/high_mean": 0.0008117913530441001, "und/clip_ratio/low_mean": 0.00011440205707913265, "und/clip_ratio/region_mean": 0.0009261934137612116, "und/kl": 0.03982456024095882, "und/loss": 0.0010868344670598162 }, { "epoch": 0.8326661329063251, "grad_norm": 0.041560448706150055, "learning_rate": 1e-05, "loss": 0.0025, "rewards/correctness_reward_func/mean": 0.28125, "rewards/correctness_reward_func/std": 0.7007648944854736, "rewards/strict_format_reward_func/mean": 0.21875, "rewards/strict_format_reward_func/std": 0.25, "step": 65, "time_profile/curr_logprobs_and_update": 0.26148034693324007, "time_profile/old_logps": 33.658358512446284, "time_profile/ref_logps": 33.74039072729647, "time_profile/reward": 0.0044120242819190025, "time_profile/text_rollout": 473.54834595508873, "train/gen_step": 4160.0, "und/advantages_abs_mean": 0.516845703125, "und/advantages_max": 2.0625, "und/advantages_mean": 0.0, "und/advantages_min": -1.6875, "und/advantages_std": 0.7237750291824341, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.75390625, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.03125, "und/kl": 0.061325576913077384, "und/loss": 0.0018601922784000635, "und/prompt/max_length": 156.0, "und/prompt/mean_length": 109.5, "und/prompt/min_length": 63.0, "und/reward": 0.5234375, "und/reward_std": 0.8563321828842163 }, { "epoch": 0.8454763811048839, "grad_norm": 0.040835775434970856, "learning_rate": 1e-05, "loss": 0.0047, "step": 66, "time_profile/curr_logprobs_and_update": 0.2613908527127933, "train/gen_step": 4224.0, "und/clip_ratio/high_mean": 0.0002187356585636735, "und/clip_ratio/low_mean": 0.0002555298342485912, "und/clip_ratio/region_mean": 0.0004742654928122647, "und/kl": 0.11611133666156093, "und/loss": 0.0033431433839723468 }, { "epoch": 0.8582866293034428, "grad_norm": 0.04072283208370209, "learning_rate": 1e-05, "loss": 0.0042, "rewards/correctness_reward_func/mean": 0.875, "rewards/correctness_reward_func/std": 1.0, "rewards/strict_format_reward_func/mean": 0.3828125, "rewards/strict_format_reward_func/std": 0.21347814798355103, "step": 67, "time_profile/curr_logprobs_and_update": 0.26157006529683713, "time_profile/old_logps": 33.639826339669526, "time_profile/ref_logps": 33.74620936065912, "time_profile/reward": 0.004473298788070679, "time_profile/text_rollout": 454.7873819489032, "train/gen_step": 4288.0, "und/advantages_abs_mean": 0.275390625, "und/advantages_max": 0.9375, "und/advantages_mean": 0.0, "und/advantages_min": -2.1875, "und/advantages_std": 0.4825730323791504, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.744140625, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.203125, "und/kl": 0.10618040141707752, "und/loss": 0.002721756522078067, "und/prompt/max_length": 53.0, "und/prompt/mean_length": 49.5, "und/prompt/min_length": 46.0, "und/reward": 1.283203125, "und/reward_std": 1.1394332647323608 }, { "epoch": 0.8710968775020016, "grad_norm": 0.023537058383226395, "learning_rate": 1e-05, "loss": 0.0019, "step": 68, "time_profile/curr_logprobs_and_update": 0.2615145814634161, "train/gen_step": 4352.0, "und/clip_ratio/high_mean": 0.0006931504431122448, "und/clip_ratio/low_mean": 0.000173399384948425, "und/clip_ratio/region_mean": 0.000866549824422691, "und/kl": 0.05065767532505561, "und/loss": 0.0008179922006092966 }, { "epoch": 0.8839071257005604, "grad_norm": 0.00545235862955451, "learning_rate": 1e-05, "loss": 0.0019, "rewards/correctness_reward_func/mean": 0.03125, "rewards/correctness_reward_func/std": 0.25, "rewards/strict_format_reward_func/mean": 0.0703125, "rewards/strict_format_reward_func/std": 0.1751912236213684, "step": 69, "time_profile/curr_logprobs_and_update": 0.261740165471565, "time_profile/old_logps": 33.70919596590102, "time_profile/ref_logps": 33.77206904441118, "time_profile/reward": 0.004499014467000961, "time_profile/text_rollout": 501.05089360289276, "train/gen_step": 4416.0, "und/advantages_abs_mean": 0.09765625, "und/advantages_max": 2.125, "und/advantages_mean": 0.0, "und/advantages_min": -0.4375, "und/advantages_std": 0.18104934692382812, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.92578125, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.53125, "und/kl": 0.04702442421694286, "und/loss": 0.0023319427573369467, "und/prompt/max_length": 313.0, "und/prompt/mean_length": 185.5, "und/prompt/min_length": 58.0, "und/reward": 0.083984375, "und/reward_std": 0.21164102852344513 }, { "epoch": 0.8967173738991193, "grad_norm": 0.015413722954690456, "learning_rate": 1e-05, "loss": 0.0029, "step": 70, "time_profile/curr_logprobs_and_update": 0.26160615020489786, "train/gen_step": 4480.0, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 9.790100375539623e-05, "und/clip_ratio/region_mean": 9.790100375539623e-05, "und/kl": 0.0741199756594142, "und/loss": 0.002338052549376357 }, { "epoch": 0.9095276220976781, "grad_norm": 0.013947661966085434, "learning_rate": 1e-05, "loss": 0.0017, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.09375, "rewards/strict_format_reward_func/std": 0.19669894874095917, "step": 71, "time_profile/curr_logprobs_and_update": 0.26171521554351784, "time_profile/old_logps": 33.66377367079258, "time_profile/ref_logps": 33.751427008770406, "time_profile/reward": 0.004377297125756741, "time_profile/text_rollout": 500.3317451989278, "train/gen_step": 4544.0, "und/advantages_abs_mean": 0.107666015625, "und/advantages_max": 0.4375, "und/advantages_mean": 0.0, "und/advantages_min": -0.4375, "und/advantages_std": 0.16422295570373535, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.857421875, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.5, "und/kl": 0.04332471443194663, "und/loss": 0.0016425758730918005, "und/prompt/max_length": 313.0, "und/prompt/mean_length": 184.0, "und/prompt/min_length": 55.0, "und/reward": 0.1005859375, "und/reward_std": 0.2006341516971588 }, { "epoch": 0.922337870296237, "grad_norm": 0.012040519155561924, "learning_rate": 1e-05, "loss": 0.0016, "step": 72, "time_profile/curr_logprobs_and_update": 0.2617163411487127, "train/gen_step": 4608.0, "und/clip_ratio/high_mean": 0.00010259100599796511, "und/clip_ratio/low_mean": 0.00037950047408230603, "und/clip_ratio/region_mean": 0.00048209148008027114, "und/kl": 0.03941097212373279, "und/loss": 0.0017510293993154846 }, { "epoch": 0.9351481184947958, "grad_norm": 0.025353815406560898, "learning_rate": 1e-05, "loss": 0.0082, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.234375, "rewards/strict_format_reward_func/std": 0.2514837086200714, "step": 73, "time_profile/curr_logprobs_and_update": 0.26159169392485637, "time_profile/old_logps": 33.65948408655822, "time_profile/ref_logps": 33.75273651909083, "time_profile/reward": 0.004401316866278648, "time_profile/text_rollout": 487.3236538115889, "train/gen_step": 4672.0, "und/advantages_abs_mean": 0.070556640625, "und/advantages_max": 2.1875, "und/advantages_mean": 0.0, "und/advantages_min": -0.6875, "und/advantages_std": 0.2424243539571762, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.818359375, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.703125, "und/kl": 0.20393660698027816, "und/loss": 0.004929899203489185, "und/prompt/max_length": 247.0, "und/prompt/mean_length": 148.0, "und/prompt/min_length": 49.0, "und/reward": 0.2666015625, "und/reward_std": 0.35626593232154846 }, { "epoch": 0.9479583666933546, "grad_norm": 0.013712815009057522, "learning_rate": 1e-05, "loss": 0.0149, "step": 74, "time_profile/curr_logprobs_and_update": 0.26143009605584666, "train/gen_step": 4736.0, "und/clip_ratio/high_mean": 0.00025846589414868504, "und/clip_ratio/low_mean": 5.278716344037093e-05, "und/clip_ratio/region_mean": 0.00031125305758905597, "und/kl": 0.36927700173691846, "und/loss": 0.0025039048924071494 }, { "epoch": 0.9607686148919136, "grad_norm": 0.009972140192985535, "learning_rate": 1e-05, "loss": 0.0136, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.21875, "rewards/strict_format_reward_func/std": 0.25, "step": 75, "time_profile/curr_logprobs_and_update": 0.26166287070373073, "time_profile/old_logps": 33.67625562660396, "time_profile/ref_logps": 33.7687153192237, "time_profile/reward": 0.0043565016239881516, "time_profile/text_rollout": 473.3151173014194, "train/gen_step": 4800.0, "und/advantages_abs_mean": 0.205810546875, "und/advantages_max": 0.4375, "und/advantages_mean": 0.0, "und/advantages_min": -0.375, "und/advantages_std": 0.22705358266830444, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.791015625, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.015625, "und/kl": 0.33891669298463967, "und/loss": 0.0008663814514875412, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 107.0, "und/prompt/min_length": 52.0, "und/reward": 0.1904296875, "und/reward_std": 0.24303650856018066 }, { "epoch": 0.9735788630904724, "grad_norm": 0.009620222263038158, "learning_rate": 1e-05, "loss": 0.0059, "step": 76, "time_profile/curr_logprobs_and_update": 0.2629899470921373, "train/gen_step": 4864.0, "und/clip_ratio/high_mean": 0.00046749321700190194, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.00046749321700190194, "und/kl": 0.1477012149625807, "und/loss": 0.0013083760859444737 }, { "epoch": 0.9863891112890312, "grad_norm": 0.010119947604835033, "learning_rate": 1e-05, "loss": 0.0234, "rewards/correctness_reward_func/mean": 0.03125, "rewards/correctness_reward_func/std": 0.25, "rewards/strict_format_reward_func/mean": 0.2578125, "rewards/strict_format_reward_func/std": 0.25185325741767883, "step": 77, "time_profile/curr_logprobs_and_update": 0.2616106412606314, "time_profile/old_logps": 33.68558882549405, "time_profile/ref_logps": 33.77748038619757, "time_profile/reward": 0.0041413139551877975, "time_profile/text_rollout": 473.13247388228774, "train/gen_step": 4928.0, "und/advantages_abs_mean": 0.216064453125, "und/advantages_max": 2.125, "und/advantages_mean": 0.0, "und/advantages_min": -0.5, "und/advantages_std": 0.2934376895427704, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.806640625, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.0625, "und/kl": 0.5857936546090059, "und/loss": 0.0010301140270740916, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 106.5, "und/prompt/min_length": 51.0, "und/reward": 0.220703125, "und/reward_std": 0.31764960289001465 }, { "epoch": 0.9991993594875901, "grad_norm": 0.012176643125712872, "learning_rate": 1e-05, "loss": 0.0303, "step": 78, "time_profile/curr_logprobs_and_update": 0.2613839999830816, "train/gen_step": 4992.0, "und/clip_ratio/high_mean": 0.00017109220061684027, "und/clip_ratio/low_mean": 0.000871925825776998, "und/clip_ratio/region_mean": 0.0010430180263938382, "und/kl": 0.7574968582484871, "und/loss": 0.0010476876539087243 }, { "epoch": 1.0, "grad_norm": 0.010883414186537266, "learning_rate": 1e-05, "loss": 0.001, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.2890625, "rewards/strict_format_reward_func/std": 0.24888142943382263, "step": 79, "time_profile/curr_logprobs_and_update": 0.26290554786100984, "time_profile/old_logps": 33.66828224621713, "time_profile/ref_logps": 33.729235981591046, "time_profile/reward": 0.004360824823379517, "time_profile/text_rollout": 456.6069682110101, "train/gen_step": 4996.0, "und/advantages_abs_mean": 0.123779296875, "und/advantages_max": 0.4375, "und/advantages_mean": 0.0, "und/advantages_min": -0.4375, "und/advantages_std": 0.17608344554901123, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.671875, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.3125, "und/kl": 0.018394303042441607, "und/loss": 0.12604861333966255, "und/prompt/max_length": 64.0, "und/prompt/mean_length": 57.0, "und/prompt/min_length": 50.0, "und/reward": 0.3056640625, "und/reward_std": 0.2439626157283783 }, { "epoch": 1.0128102481985588, "grad_norm": 0.011202736757695675, "learning_rate": 1e-05, "loss": 0.0026, "step": 80, "time_profile/curr_logprobs_and_update": 0.26144698687130585, "train/gen_step": 5060.0, "und/clip_ratio/high_mean": 0.00020480685998336412, "und/clip_ratio/low_mean": 0.0010958394632325508, "und/clip_ratio/region_mean": 0.001300646319577936, "und/kl": 0.06380279999575578, "und/loss": 0.001128781703300774 }, { "epoch": 1.0256204963971176, "grad_norm": 0.011037485674023628, "learning_rate": 1e-05, "loss": -0.0013, "rewards/correctness_reward_func/mean": 0.96875, "rewards/correctness_reward_func/std": 1.0074130296707153, "rewards/strict_format_reward_func/mean": 0.296875, "rewards/strict_format_reward_func/std": 0.24750742316246033, "step": 81, "time_profile/curr_logprobs_and_update": 0.2614856250002049, "time_profile/old_logps": 33.64920091070235, "time_profile/ref_logps": 33.70459945779294, "time_profile/reward": 0.004259305074810982, "time_profile/text_rollout": 454.86890551261604, "train/gen_step": 5124.0, "und/advantages_abs_mean": 0.21484375, "und/advantages_max": 0.9375, "und/advantages_mean": 0.0, "und/advantages_min": -2.1875, "und/advantages_std": 0.4081484079360962, "und/clip_ratio/high_mean": 0.0003931849460059311, "und/clip_ratio/low_mean": 0.0013027411841903813, "und/clip_ratio/region_mean": 0.0016959261301963124, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.779296875, "und/completion/min_length": 222.0, "und/frac_reward_zero_std": 0.328125, "und/kl": 0.06465533758455422, "und/loss": -0.005637970802126802, "und/prompt/max_length": 58.0, "und/prompt/mean_length": 51.0, "und/prompt/min_length": 44.0, "und/reward": 1.2578125, "und/reward_std": 1.1828593015670776 }, { "epoch": 1.0384307445956766, "grad_norm": 0.012660922482609749, "learning_rate": 1e-05, "loss": 0.0052, "step": 82, "time_profile/curr_logprobs_and_update": 0.2613754039048217, "train/gen_step": 5188.0, "und/clip_ratio/high_mean": 0.00011789133350248449, "und/clip_ratio/low_mean": 0.0001570866115798708, "und/clip_ratio/region_mean": 0.0002749779450823553, "und/kl": 0.13132090101134963, "und/loss": 0.0017235139853255532 }, { "epoch": 1.0512409927942354, "grad_norm": 0.04931657388806343, "learning_rate": 1e-05, "loss": 0.0348, "rewards/correctness_reward_func/mean": 0.3125, "rewards/correctness_reward_func/std": 0.7319250702857971, "rewards/strict_format_reward_func/mean": 0.390625, "rewards/strict_format_reward_func/std": 0.2083333432674408, "step": 83, "time_profile/curr_logprobs_and_update": 0.2614861790498253, "time_profile/old_logps": 33.64623137563467, "time_profile/ref_logps": 33.70282917190343, "time_profile/reward": 0.004459716379642487, "time_profile/text_rollout": 454.369918580167, "train/gen_step": 5252.0, "und/advantages_abs_mean": 0.470703125, "und/advantages_max": 1.75, "und/advantages_mean": 0.0, "und/advantages_min": -1.4375, "und/advantages_std": 0.6308677792549133, "und/clip_ratio/high_mean": 0.00037126274764887057, "und/clip_ratio/low_mean": 0.0008432684480794705, "und/clip_ratio/region_mean": 0.0012145311957283411, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.71875, "und/completion/min_length": 220.0, "und/frac_reward_zero_std": 0.046875, "und/kl": 0.8189952824468492, "und/loss": 0.002741719154414568, "und/prompt/max_length": 51.0, "und/prompt/mean_length": 47.5, "und/prompt/min_length": 44.0, "und/reward": 0.689453125, "und/reward_std": 0.7827932238578796 }, { "epoch": 1.0640512409927942, "grad_norm": 0.0498473197221756, "learning_rate": 1e-05, "loss": 0.0023, "step": 84, "time_profile/curr_logprobs_and_update": 0.26133355250931345, "train/gen_step": 5316.0, "und/clip_ratio/high_mean": 0.0004264039162080735, "und/clip_ratio/low_mean": 0.0006009412281855475, "und/clip_ratio/region_mean": 0.0010273451480315998, "und/kl": 0.060061977404984646, "und/loss": 0.002192927919622889 }, { "epoch": 1.076861489191353, "grad_norm": 0.037328220903873444, "learning_rate": 1e-05, "loss": 0.0067, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.4921875, "rewards/strict_format_reward_func/std": 0.0625, "step": 85, "time_profile/curr_logprobs_and_update": 0.26134839945007116, "time_profile/old_logps": 33.615224280394614, "time_profile/ref_logps": 33.697587930597365, "time_profile/reward": 0.004523534327745438, "time_profile/text_rollout": 454.07083107624203, "train/gen_step": 5380.0, "und/advantages_abs_mean": 0.103759765625, "und/advantages_max": 1.8125, "und/advantages_mean": 0.0, "und/advantages_min": -0.6875, "und/advantages_std": 0.28764939308166504, "und/clip_ratio/high_mean": 0.0008836273773340508, "und/clip_ratio/low_mean": 0.001003391091217054, "und/clip_ratio/region_mean": 0.0018870184685511049, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.63671875, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.625, "und/kl": 0.05825820894096978, "und/loss": 0.002590392931324459, "und/prompt/max_length": 47.0, "und/prompt/mean_length": 45.0, "und/prompt/min_length": 43.0, "und/reward": 0.5224609375, "und/reward_std": 0.30963554978370667 }, { "epoch": 1.0896717373899119, "grad_norm": 0.029800761491060257, "learning_rate": 1e-05, "loss": 0.0081, "step": 86, "time_profile/curr_logprobs_and_update": 0.2612544225266902, "train/gen_step": 5444.0, "und/clip_ratio/high_mean": 9.385726662003435e-05, "und/clip_ratio/low_mean": 9.646531907492317e-05, "und/clip_ratio/region_mean": 0.00019032258569495752, "und/kl": 0.20097231600084342, "und/loss": 0.0032604012124011206 }, { "epoch": 1.1024819855884709, "grad_norm": 0.03139861673116684, "learning_rate": 1e-05, "loss": 0.0314, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.09375, "rewards/strict_format_reward_func/std": 0.19669894874095917, "step": 87, "time_profile/curr_logprobs_and_update": 0.26139444966975134, "time_profile/old_logps": 33.700514083728194, "time_profile/ref_logps": 33.78526249341667, "time_profile/reward": 0.004218182526528835, "time_profile/text_rollout": 518.5401958124712, "train/gen_step": 5508.0, "und/advantages_abs_mean": 0.0966796875, "und/advantages_max": 0.4375, "und/advantages_mean": 0.0, "und/advantages_min": -0.3125, "und/advantages_std": 0.1556188315153122, "und/clip_ratio/high_mean": 0.00026991102640749887, "und/clip_ratio/low_mean": 0.0002991936562466435, "und/clip_ratio/region_mean": 0.0005691046826541424, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.720703125, "und/completion/min_length": 159.0, "und/frac_reward_zero_std": 0.5, "und/kl": 0.7756888958901982, "und/loss": 0.01109763615556858, "und/prompt/max_length": 313.0, "und/prompt/mean_length": 237.5, "und/prompt/min_length": 162.0, "und/reward": 0.080078125, "und/reward_std": 0.18355479836463928 }, { "epoch": 1.1152922337870297, "grad_norm": 0.007225779816508293, "learning_rate": 1e-05, "loss": 0.0023, "step": 88, "time_profile/curr_logprobs_and_update": 0.2618225736368913, "train/gen_step": 5572.0, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/kl": 0.05710822065520915, "und/loss": 0.0013941282181804127 }, { "epoch": 1.1281024819855885, "grad_norm": 0.006840178277343512, "learning_rate": 1e-05, "loss": 0.0449, "rewards/correctness_reward_func/mean": 0.28125, "rewards/correctness_reward_func/std": 0.7007648944854736, "rewards/strict_format_reward_func/mean": 0.2109375, "rewards/strict_format_reward_func/std": 0.24888142943382263, "step": 89, "time_profile/curr_logprobs_and_update": 0.2619360370154027, "time_profile/old_logps": 33.66616447176784, "time_profile/ref_logps": 33.762949594296515, "time_profile/reward": 0.004555100575089455, "time_profile/text_rollout": 499.7487023388967, "train/gen_step": 5636.0, "und/advantages_abs_mean": 0.455078125, "und/advantages_max": 1.8125, "und/advantages_mean": 0.0, "und/advantages_min": -1.9375, "und/advantages_std": 0.6984922289848328, "und/clip_ratio/high_mean": 0.00011725750300684012, "und/clip_ratio/low_mean": 0.0001583614903211128, "und/clip_ratio/region_mean": 0.0002756189933279529, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.771484375, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.46875, "und/kl": 1.232714807476441, "und/loss": -0.006696377290722921, "und/prompt/max_length": 313.0, "und/prompt/mean_length": 180.0, "und/prompt/min_length": 47.0, "und/reward": 0.734375, "und/reward_std": 1.0485988855361938 }, { "epoch": 1.1409127301841473, "grad_norm": 0.006123845931142569, "learning_rate": 1e-05, "loss": 0.004, "step": 90, "time_profile/curr_logprobs_and_update": 0.26148606633068994, "train/gen_step": 5700.0, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.00026530873219599016, "und/clip_ratio/region_mean": 0.00026530873219599016, "und/kl": 0.09945900924503803, "und/loss": 0.004926550779600802 }, { "epoch": 1.153722978382706, "grad_norm": 0.009329823777079582, "learning_rate": 1e-05, "loss": 0.0647, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.3046875, "rewards/strict_format_reward_func/std": 0.24587368965148926, "step": 91, "time_profile/curr_logprobs_and_update": 0.26152575001469813, "time_profile/old_logps": 33.65729128010571, "time_profile/ref_logps": 33.73677711561322, "time_profile/reward": 0.004335583187639713, "time_profile/text_rollout": 472.53139427211136, "train/gen_step": 5764.0, "und/advantages_abs_mean": 0.19921875, "und/advantages_max": 1.8125, "und/advantages_mean": 0.0, "und/advantages_min": -0.6875, "und/advantages_std": 0.35682472586631775, "und/clip_ratio/high_mean": 0.0003500454331515357, "und/clip_ratio/low_mean": 0.00026731862817541696, "und/clip_ratio/region_mean": 0.0006173640649649315, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.72265625, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.265625, "und/kl": 1.6728466484637465, "und/loss": 0.006491393034089299, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 104.0, "und/prompt/min_length": 46.0, "und/reward": 0.359375, "und/reward_std": 0.4551832377910614 }, { "epoch": 1.1665332265812651, "grad_norm": 3.300291061401367, "learning_rate": 1e-05, "loss": 0.021, "step": 92, "time_profile/curr_logprobs_and_update": 0.2615345583180897, "train/gen_step": 5828.0, "und/clip_ratio/high_mean": 0.00015658686243114062, "und/clip_ratio/low_mean": 0.000412728051742306, "und/clip_ratio/region_mean": 0.0005693149105354678, "und/kl": 0.5270771795330802, "und/loss": 0.021786472902746823 }, { "epoch": 1.179343474779824, "grad_norm": 0.04499354213476181, "learning_rate": 1e-05, "loss": 0.0101, "rewards/correctness_reward_func/mean": 0.78125, "rewards/correctness_reward_func/std": 0.983494758605957, "rewards/strict_format_reward_func/mean": 0.359375, "rewards/strict_format_reward_func/std": 0.22658175230026245, "step": 93, "time_profile/curr_logprobs_and_update": 0.2617357307317434, "time_profile/old_logps": 33.68263782840222, "time_profile/ref_logps": 33.77571779489517, "time_profile/reward": 0.004069678485393524, "time_profile/text_rollout": 456.24606967996806, "train/gen_step": 5892.0, "und/advantages_abs_mean": 0.408447265625, "und/advantages_max": 0.875, "und/advantages_mean": 0.0, "und/advantages_min": -2.1875, "und/advantages_std": 0.5925272703170776, "und/clip_ratio/high_mean": 0.0006427463522413746, "und/clip_ratio/low_mean": 0.0003597673749027308, "und/clip_ratio/region_mean": 0.0010025137307820842, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.634765625, "und/completion/min_length": 173.0, "und/frac_reward_zero_std": 0.078125, "und/kl": 0.16817912326951046, "und/loss": 0.0022755718414586568, "und/prompt/max_length": 65.0, "und/prompt/mean_length": 54.0, "und/prompt/min_length": 43.0, "und/reward": 1.1474609375, "und/reward_std": 1.1275041103363037 }, { "epoch": 1.1921537229783827, "grad_norm": 0.06060933321714401, "learning_rate": 1e-05, "loss": 0.0042, "step": 94, "time_profile/curr_logprobs_and_update": 0.26131482362688985, "train/gen_step": 5956.0, "und/clip_ratio/high_mean": 0.0019653168055810966, "und/clip_ratio/low_mean": 0.00016475993470521644, "und/clip_ratio/region_mean": 0.002130076743924292, "und/kl": 0.10070258447376546, "und/loss": 0.005654156702803448 }, { "epoch": 1.2049639711769415, "grad_norm": 0.03084016963839531, "learning_rate": 1e-05, "loss": 0.0413, "rewards/correctness_reward_func/mean": 0.03125, "rewards/correctness_reward_func/std": 0.25, "rewards/strict_format_reward_func/mean": 0.265625, "rewards/strict_format_reward_func/std": 0.2514837086200714, "step": 95, "time_profile/curr_logprobs_and_update": 0.26137867687793914, "time_profile/old_logps": 33.66459304280579, "time_profile/ref_logps": 33.77252321410924, "time_profile/reward": 0.004362174309790134, "time_profile/text_rollout": 486.8325478192419, "train/gen_step": 6020.0, "und/advantages_abs_mean": 0.0693359375, "und/advantages_max": 2.1875, "und/advantages_mean": 0.0, "und/advantages_min": -0.625, "und/advantages_std": 0.27044573426246643, "und/clip_ratio/high_mean": 0.0046464357401418965, "und/clip_ratio/low_mean": 0.0008429612207692116, "und/clip_ratio/region_mean": 0.005489396939083235, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.8828125, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.8125, "und/kl": 1.0781144749053055, "und/loss": 0.00424630767565759, "und/prompt/max_length": 247.0, "und/prompt/mean_length": 145.0, "und/prompt/min_length": 43.0, "und/reward": 0.287109375, "und/reward_std": 0.3735242784023285 }, { "epoch": 1.2177742193755003, "grad_norm": 0.008688083849847317, "learning_rate": 1e-05, "loss": 0.0111, "step": 96, "time_profile/curr_logprobs_and_update": 0.2616057716368232, "train/gen_step": 6084.0, "und/clip_ratio/high_mean": 0.00013542917804443277, "und/clip_ratio/low_mean": 4.882812572759576e-05, "und/clip_ratio/region_mean": 0.00018425730377202854, "und/kl": 0.27596294990507886, "und/loss": 0.002798294794956746 }, { "epoch": 1.2305844675740594, "grad_norm": 0.003840341465547681, "learning_rate": 1e-05, "loss": 0.0055, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0546875, "rewards/strict_format_reward_func/std": 0.15728822350502014, "step": 97, "time_profile/curr_logprobs_and_update": 0.26164151464763563, "time_profile/old_logps": 33.694718721322715, "time_profile/ref_logps": 33.787934163585305, "time_profile/reward": 0.004452788271009922, "time_profile/text_rollout": 518.2917092712596, "train/gen_step": 6148.0, "und/advantages_abs_mean": 0.088623046875, "und/advantages_max": 0.4375, "und/advantages_mean": 0.0, "und/advantages_min": -0.3125, "und/advantages_std": 0.14899368584156036, "und/clip_ratio/high_mean": 0.000445808833319461, "und/clip_ratio/low_mean": 0.0001805975625757128, "und/clip_ratio/region_mean": 0.0006264063958951738, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.91015625, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.546875, "und/kl": 0.08030579777550884, "und/loss": 0.008490518508324385, "und/prompt/max_length": 313.0, "und/prompt/mean_length": 237.5, "und/prompt/min_length": 162.0, "und/reward": 0.0673828125, "und/reward_std": 0.1709035038948059 }, { "epoch": 1.2433947157726182, "grad_norm": 0.002296620514243841, "learning_rate": 1e-05, "loss": 0.0021, "step": 98, "time_profile/curr_logprobs_and_update": 0.2616005388117628, "train/gen_step": 6212.0, "und/clip_ratio/high_mean": 0.0002009032541536726, "und/clip_ratio/low_mean": 5.5803571740398183e-05, "und/clip_ratio/region_mean": 0.0002567068258940708, "und/kl": 0.05221230116148945, "und/loss": 0.0016822464920096536 }, { "epoch": 1.256204963971177, "grad_norm": 0.0028974718879908323, "learning_rate": 1e-05, "loss": 0.0042, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0546875, "rewards/strict_format_reward_func/std": 0.15728822350502014, "step": 99, "time_profile/curr_logprobs_and_update": 0.26193443170632236, "time_profile/old_logps": 33.68688790965825, "time_profile/ref_logps": 33.75524262525141, "time_profile/reward": 0.004406670108437538, "time_profile/text_rollout": 518.5918508702889, "train/gen_step": 6276.0, "und/advantages_abs_mean": 0.095458984375, "und/advantages_max": 0.4375, "und/advantages_mean": 0.0, "und/advantages_min": -0.3125, "und/advantages_std": 0.1546332687139511, "und/clip_ratio/high_mean": 0.00018194289805251174, "und/clip_ratio/low_mean": 4.340277882874943e-05, "und/clip_ratio/region_mean": 0.00022534567688126117, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.91796875, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.515625, "und/kl": 0.10376544366590679, "und/loss": 0.0064854817031232415, "und/prompt/max_length": 313.0, "und/prompt/mean_length": 237.5, "und/prompt/min_length": 162.0, "und/reward": 0.0732421875, "und/reward_std": 0.17696848511695862 }, { "epoch": 1.2690152121697358, "grad_norm": 0.002257066546007991, "learning_rate": 1e-05, "loss": 0.0048, "step": 100, "time_profile/curr_logprobs_and_update": 0.26184689076035284, "train/gen_step": 6340.0, "und/clip_ratio/high_mean": 5.425347262644209e-05, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 5.425347262644209e-05, "und/kl": 0.12037181136838626, "und/loss": 0.0016251774967486199 }, { "epoch": 1.2818254603682946, "grad_norm": 0.04131701961159706, "learning_rate": 1e-05, "loss": 0.0128, "rewards/correctness_reward_func/mean": 0.28125, "rewards/correctness_reward_func/std": 0.7007648944854736, "rewards/strict_format_reward_func/mean": 0.3046875, "rewards/strict_format_reward_func/std": 0.24587368965148926, "step": 101, "time_profile/curr_logprobs_and_update": 0.2618017203931231, "time_profile/old_logps": 33.654254213906825, "time_profile/ref_logps": 33.73097015917301, "time_profile/reward": 0.004532042890787125, "time_profile/text_rollout": 455.13132797554135, "train/gen_step": 6404.0, "und/advantages_abs_mean": 0.5234375, "und/advantages_max": 1.875, "und/advantages_mean": 0.0, "und/advantages_min": -1.5, "und/advantages_std": 0.676927924156189, "und/clip_ratio/high_mean": 0.00020775113807758316, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.00020775113807758316, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.73046875, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.015625, "und/kl": 0.11258987631299533, "und/loss": 0.01647166971235947, "und/prompt/max_length": 56.0, "und/prompt/mean_length": 52.5, "und/prompt/min_length": 49.0, "und/reward": 0.7099609375, "und/reward_std": 0.905170202255249 }, { "epoch": 1.2946357085668536, "grad_norm": 0.05473562330007553, "learning_rate": 1e-05, "loss": 0.0355, "step": 102, "time_profile/curr_logprobs_and_update": 0.261388782368158, "train/gen_step": 6468.0, "und/clip_ratio/high_mean": 0.0005066352496214677, "und/clip_ratio/low_mean": 9.820867489906959e-05, "und/clip_ratio/region_mean": 0.0006048439317964949, "und/kl": 0.8876906962541398, "und/loss": 0.001254691684152931 }, { "epoch": 1.3074459567654122, "grad_norm": 0.034072067588567734, "learning_rate": 1e-05, "loss": -0.0061, "rewards/correctness_reward_func/mean": 0.0625, "rewards/correctness_reward_func/std": 0.35073620080947876, "rewards/strict_format_reward_func/mean": 0.2265625, "rewards/strict_format_reward_func/std": 0.250866562128067, "step": 103, "time_profile/curr_logprobs_and_update": 0.26150604666327126, "time_profile/old_logps": 33.661563923582435, "time_profile/ref_logps": 33.73078432120383, "time_profile/reward": 0.004420598968863487, "time_profile/text_rollout": 499.30165772140026, "train/gen_step": 6532.0, "und/advantages_abs_mean": 0.116455078125, "und/advantages_max": 1.9375, "und/advantages_mean": 0.0, "und/advantages_min": -0.9375, "und/advantages_std": 0.3238520324230194, "und/clip_ratio/high_mean": 0.0007794650773575995, "und/clip_ratio/low_mean": 0.0006103413725213613, "und/clip_ratio/region_mean": 0.001389806446240982, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.703125, "und/completion/min_length": 190.0, "und/frac_reward_zero_std": 0.65625, "und/kl": 0.0528246548055904, "und/loss": -0.01438412336096917, "und/prompt/max_length": 313.0, "und/prompt/mean_length": 178.0, "und/prompt/min_length": 43.0, "und/reward": 0.2900390625, "und/reward_std": 0.45152318477630615 }, { "epoch": 1.3202562049639712, "grad_norm": 0.005107598844915628, "learning_rate": 1e-05, "loss": 0.0064, "step": 104, "time_profile/curr_logprobs_and_update": 0.26163111737696454, "train/gen_step": 6596.0, "und/clip_ratio/high_mean": 5.918560782447457e-05, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 5.918560782447457e-05, "und/kl": 0.15930140123236924, "und/loss": 0.0028715015233444774 }, { "epoch": 1.33306645316253, "grad_norm": 0.028493041172623634, "learning_rate": 1e-05, "loss": 0.0042, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.1953125, "rewards/strict_format_reward_func/std": 0.24587368965148926, "step": 105, "time_profile/curr_logprobs_and_update": 0.2615614668466151, "time_profile/old_logps": 33.699232946150005, "time_profile/ref_logps": 33.78243537712842, "time_profile/reward": 0.004495236091315746, "time_profile/text_rollout": 491.3853238923475, "train/gen_step": 6660.0, "und/advantages_abs_mean": 0.205078125, "und/advantages_max": 0.4375, "und/advantages_mean": 0.0, "und/advantages_min": -0.3125, "und/advantages_std": 0.226649209856987, "und/clip_ratio/high_mean": 0.00030362879260792397, "und/clip_ratio/low_mean": 9.527438669465482e-05, "und/clip_ratio/region_mean": 0.0003989031793025788, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.7734375, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.015625, "und/kl": 0.10507874877657741, "und/loss": -0.0043832145364604, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 162.0, "und/prompt/min_length": 162.0, "und/reward": 0.1640625, "und/reward_std": 0.23499490320682526 }, { "epoch": 1.3458767013610888, "grad_norm": 0.009856931865215302, "learning_rate": 1e-05, "loss": 0.0113, "step": 106, "time_profile/curr_logprobs_and_update": 0.2615844376850873, "train/gen_step": 6724.0, "und/clip_ratio/high_mean": 0.00010460711928317323, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.00010460711928317323, "und/kl": 0.2839002070031711, "und/loss": 0.0006757988594472408 }, { "epoch": 1.3586869495596476, "grad_norm": 0.013232600875198841, "learning_rate": 1e-05, "loss": 0.0067, "rewards/correctness_reward_func/mean": 0.03125, "rewards/correctness_reward_func/std": 0.25, "rewards/strict_format_reward_func/mean": 0.2578125, "rewards/strict_format_reward_func/std": 0.25185325741767883, "step": 107, "time_profile/curr_logprobs_and_update": 0.26170188521791715, "time_profile/old_logps": 33.66488418634981, "time_profile/ref_logps": 33.74126262869686, "time_profile/reward": 0.004568587988615036, "time_profile/text_rollout": 455.8338515488431, "train/gen_step": 6788.0, "und/advantages_abs_mean": 0.17236328125, "und/advantages_max": 2.0, "und/advantages_mean": 0.0, "und/advantages_min": -0.5, "und/advantages_std": 0.2258382886648178, "und/clip_ratio/high_mean": 0.0003017742856172845, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0003017742856172845, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.767578125, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.140625, "und/kl": 0.20220415030053118, "und/loss": 0.008677718930357514, "und/prompt/max_length": 58.0, "und/prompt/mean_length": 55.5, "und/prompt/min_length": 53.0, "und/reward": 0.302734375, "und/reward_std": 0.26385733485221863 }, { "epoch": 1.3714971977582064, "grad_norm": 0.01322892215102911, "learning_rate": 1e-05, "loss": 0.002, "step": 108, "time_profile/curr_logprobs_and_update": 0.2693607303954195, "train/gen_step": 6852.0, "und/clip_ratio/high_mean": 0.0003047961799893528, "und/clip_ratio/low_mean": 0.0001023065487970598, "und/clip_ratio/region_mean": 0.0004071027287864126, "und/kl": 0.04968085115979193, "und/loss": 0.0027706179830602196 }, { "epoch": 1.3843074459567655, "grad_norm": 0.012319354340434074, "learning_rate": 1e-05, "loss": 0.0078, "rewards/correctness_reward_func/mean": 0.375, "rewards/correctness_reward_func/std": 0.7867957949638367, "rewards/strict_format_reward_func/mean": 0.375, "rewards/strict_format_reward_func/std": 0.2182178944349289, "step": 109, "time_profile/curr_logprobs_and_update": 0.2616202123608673, "time_profile/old_logps": 33.62943078484386, "time_profile/ref_logps": 33.73900862503797, "time_profile/reward": 0.004435631446540356, "time_profile/text_rollout": 454.5818900093436, "train/gen_step": 6916.0, "und/advantages_abs_mean": 0.563232421875, "und/advantages_max": 1.875, "und/advantages_mean": 0.0, "und/advantages_min": -1.6875, "und/advantages_std": 0.72254878282547, "und/clip_ratio/high_mean": 0.0008765930215304252, "und/clip_ratio/low_mean": 9.084302291739732e-05, "und/clip_ratio/region_mean": 0.0009674360444478225, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.5234375, "und/completion/min_length": 180.0, "und/frac_reward_zero_std": 0.015625, "und/kl": 0.10922401638526935, "und/loss": 0.014170625921906321, "und/prompt/max_length": 51.0, "und/prompt/mean_length": 48.5, "und/prompt/min_length": 46.0, "und/reward": 0.7587890625, "und/reward_std": 0.916750431060791 }, { "epoch": 1.3971176941553243, "grad_norm": 0.014209937304258347, "learning_rate": 1e-05, "loss": 0.0026, "step": 110, "time_profile/curr_logprobs_and_update": 0.2613295341434423, "train/gen_step": 6980.0, "und/clip_ratio/high_mean": 0.00018801813712343574, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.00018801813712343574, "und/kl": 0.06319543437712127, "und/loss": 0.0012067200150340796 }, { "epoch": 1.409927942353883, "grad_norm": 0.00816685613244772, "learning_rate": 1e-05, "loss": 0.0138, "rewards/correctness_reward_func/mean": 0.9375, "rewards/correctness_reward_func/std": 1.0059348344802856, "rewards/strict_format_reward_func/mean": 0.25, "rewards/strict_format_reward_func/std": 0.2519763112068176, "step": 111, "time_profile/curr_logprobs_and_update": 0.26151229023525957, "time_profile/old_logps": 33.65623056329787, "time_profile/ref_logps": 33.743903799913824, "time_profile/reward": 0.004555624909698963, "time_profile/text_rollout": 499.53031921666116, "train/gen_step": 7044.0, "und/advantages_abs_mean": 0.22607421875, "und/advantages_max": 2.1875, "und/advantages_mean": 0.0, "und/advantages_min": -2.1875, "und/advantages_std": 0.4984687864780426, "und/clip_ratio/high_mean": 0.00038183126162039116, "und/clip_ratio/low_mean": 0.00011579119382076897, "und/clip_ratio/region_mean": 0.0004976224590791389, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.775390625, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.640625, "und/kl": 0.3358783017611131, "und/loss": 0.009991740759346612, "und/prompt/max_length": 313.0, "und/prompt/mean_length": 178.5, "und/prompt/min_length": 44.0, "und/reward": 1.1123046875, "und/reward_std": 1.224579095840454 }, { "epoch": 1.4227381905524419, "grad_norm": 0.0051344470120966434, "learning_rate": 1e-05, "loss": 0.0021, "step": 112, "time_profile/curr_logprobs_and_update": 0.2616101046296535, "train/gen_step": 7108.0, "und/clip_ratio/high_mean": 5.139802669873461e-05, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 5.139802669873461e-05, "und/kl": 0.05211045581381768, "und/loss": 0.0019724031756140903 }, { "epoch": 1.4355484387510007, "grad_norm": 0.05978088825941086, "learning_rate": 1e-05, "loss": 0.0074, "rewards/correctness_reward_func/mean": 1.4375, "rewards/correctness_reward_func/std": 0.9063270092010498, "rewards/strict_format_reward_func/mean": 0.4921875, "rewards/strict_format_reward_func/std": 0.0625, "step": 113, "time_profile/curr_logprobs_and_update": 0.2615887124411529, "time_profile/old_logps": 33.653056289069355, "time_profile/ref_logps": 33.754895557649434, "time_profile/reward": 0.0044929469004273415, "time_profile/text_rollout": 454.0134041188285, "train/gen_step": 7172.0, "und/advantages_abs_mean": 0.576416015625, "und/advantages_max": 1.8125, "und/advantages_mean": 0.0, "und/advantages_min": -2.1875, "und/advantages_std": 0.769360363483429, "und/clip_ratio/high_mean": 5.008012885809876e-05, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 5.008012885809876e-05, "und/completion/max_length": 255.0, "und/completion/mean_length": 253.978515625, "und/completion/min_length": 151.0, "und/frac_reward_zero_std": 0.234375, "und/kl": 0.07507353127584793, "und/loss": 0.0015328295562540006, "und/prompt/max_length": 49.0, "und/prompt/mean_length": 46.5, "und/prompt/min_length": 44.0, "und/reward": 1.9208984375, "und/reward_std": 0.9253405928611755 }, { "epoch": 1.4483586869495597, "grad_norm": 0.09742525219917297, "learning_rate": 1e-05, "loss": 0.0053, "step": 114, "time_profile/curr_logprobs_and_update": 0.2612951010960387, "train/gen_step": 7236.0, "und/clip_ratio/high_mean": 0.00040703625199967064, "und/clip_ratio/low_mean": 0.0002526224052417092, "und/clip_ratio/region_mean": 0.0006596586572413798, "und/kl": 0.13804292841814458, "und/loss": 0.013488775107816764 }, { "epoch": 1.4611689351481185, "grad_norm": 0.04351665452122688, "learning_rate": 1e-05, "loss": -0.002, "rewards/correctness_reward_func/mean": 0.0625, "rewards/correctness_reward_func/std": 0.35073620080947876, "rewards/strict_format_reward_func/mean": 0.3359375, "rewards/strict_format_reward_func/std": 0.2366211861371994, "step": 115, "time_profile/curr_logprobs_and_update": 0.26138295279815793, "time_profile/old_logps": 33.6624674256891, "time_profile/ref_logps": 33.752434914931655, "time_profile/reward": 0.004297954961657524, "time_profile/text_rollout": 454.55648574698716, "train/gen_step": 7300.0, "und/advantages_abs_mean": 0.22314453125, "und/advantages_max": 1.8125, "und/advantages_mean": 0.0, "und/advantages_min": -0.9375, "und/advantages_std": 0.378530353307724, "und/clip_ratio/high_mean": 0.00232754351964104, "und/clip_ratio/low_mean": 0.00021809619283885695, "und/clip_ratio/region_mean": 0.0025456397052039392, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.7421875, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.21875, "und/kl": 0.07022856344701722, "und/loss": -0.011109988249302205, "und/prompt/max_length": 52.0, "und/prompt/mean_length": 48.5, "und/prompt/min_length": 45.0, "und/reward": 0.4130859375, "und/reward_std": 0.4497930109500885 }, { "epoch": 1.4739791833466773, "grad_norm": 0.041889388114213943, "learning_rate": 1e-05, "loss": 0.0074, "step": 116, "time_profile/curr_logprobs_and_update": 0.26131885705399327, "train/gen_step": 7364.0, "und/clip_ratio/high_mean": 0.00015886059190961532, "und/clip_ratio/low_mean": 9.917301213135943e-05, "und/clip_ratio/region_mean": 0.00025803360404097475, "und/kl": 0.1826191512591322, "und/loss": 0.003956302835831593 }, { "epoch": 1.4867894315452361, "grad_norm": 0.03012249618768692, "learning_rate": 1e-05, "loss": 0.0046, "rewards/correctness_reward_func/mean": 0.03125, "rewards/correctness_reward_func/std": 0.25, "rewards/strict_format_reward_func/mean": 0.265625, "rewards/strict_format_reward_func/std": 0.2514837086200714, "step": 117, "time_profile/curr_logprobs_and_update": 0.2616950735828141, "time_profile/old_logps": 33.65449043828994, "time_profile/ref_logps": 33.74488543625921, "time_profile/reward": 0.0031747594475746155, "time_profile/text_rollout": 486.8246115138754, "train/gen_step": 7428.0, "und/advantages_abs_mean": 0.090576171875, "und/advantages_max": 2.1875, "und/advantages_mean": 0.0, "und/advantages_min": -0.4375, "und/advantages_std": 0.304784893989563, "und/clip_ratio/high_mean": 0.0004131392852286808, "und/clip_ratio/low_mean": 0.00020289487292757258, "und/clip_ratio/region_mean": 0.0006160341581562534, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.845703125, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.703125, "und/kl": 0.14989926843554713, "und/loss": 0.0282161407434387, "und/prompt/max_length": 247.0, "und/prompt/mean_length": 145.0, "und/prompt/min_length": 43.0, "und/reward": 0.2861328125, "und/reward_std": 0.37623000144958496 }, { "epoch": 1.499599679743795, "grad_norm": 0.006430523935705423, "learning_rate": 1e-05, "loss": 0.0048, "step": 118, "time_profile/curr_logprobs_and_update": 0.26145954702224117, "train/gen_step": 7492.0, "und/clip_ratio/high_mean": 0.0002027803311648313, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0002027803311648313, "und/kl": 0.12073310048435815, "und/loss": 0.002307371724555196 }, { "epoch": 1.512409927942354, "grad_norm": 0.00652286084368825, "learning_rate": 1e-05, "loss": 0.0079, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.1328125, "rewards/strict_format_reward_func/std": 0.22257846593856812, "step": 119, "time_profile/curr_logprobs_and_update": 0.2616856778477086, "time_profile/old_logps": 33.690122249536216, "time_profile/ref_logps": 33.760620589368045, "time_profile/reward": 0.004584756679832935, "time_profile/text_rollout": 491.3738481281325, "train/gen_step": 7556.0, "und/advantages_abs_mean": 0.17724609375, "und/advantages_max": 0.4375, "und/advantages_mean": 0.0, "und/advantages_min": -0.375, "und/advantages_std": 0.210708886384964, "und/clip_ratio/high_mean": 0.0003724069974850863, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0003724069974850863, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.8046875, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.0625, "und/kl": 0.11169646086636931, "und/loss": 0.006852565979897918, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 162.0, "und/prompt/min_length": 162.0, "und/reward": 0.14453125, "und/reward_std": 0.22688518464565277 }, { "epoch": 1.5252201761409128, "grad_norm": 0.010090266354382038, "learning_rate": 1e-05, "loss": 0.0047, "step": 120, "time_profile/curr_logprobs_and_update": 0.2616492405504687, "train/gen_step": 7620.0, "und/clip_ratio/high_mean": 0.00010801373355207033, "und/clip_ratio/low_mean": 8.448401058558375e-05, "und/clip_ratio/region_mean": 0.00019249774413765408, "und/kl": 0.11743137614394072, "und/loss": 0.004808933234471624 }, { "epoch": 1.5380304243394716, "grad_norm": 0.010744316503405571, "learning_rate": 1e-05, "loss": -0.0028, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.1328125, "rewards/strict_format_reward_func/std": 0.22257846593856812, "step": 121, "time_profile/curr_logprobs_and_update": 0.2618774699512869, "time_profile/old_logps": 33.68734922725707, "time_profile/ref_logps": 33.784588277339935, "time_profile/reward": 0.004522095434367657, "time_profile/text_rollout": 491.46461437735707, "train/gen_step": 7684.0, "und/advantages_abs_mean": 0.173095703125, "und/advantages_max": 0.4375, "und/advantages_mean": 0.0, "und/advantages_min": -0.3125, "und/advantages_std": 0.20822730660438538, "und/clip_ratio/high_mean": 0.00026392990548629314, "und/clip_ratio/low_mean": 0.00010926573668257333, "und/clip_ratio/region_mean": 0.00037319564216886647, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.8203125, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.078125, "und/kl": 0.09060025405779015, "und/loss": -0.003209088767432533, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 162.0, "und/prompt/min_length": 162.0, "und/reward": 0.1337890625, "und/reward_std": 0.2215646654367447 }, { "epoch": 1.5508406725380304, "grad_norm": 0.9860634803771973, "learning_rate": 1e-05, "loss": 0.0568, "step": 122, "time_profile/curr_logprobs_and_update": 0.2615589259658009, "train/gen_step": 7748.0, "und/clip_ratio/high_mean": 0.0002418396543362178, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0002418396543362178, "und/kl": 1.4184919325416558, "und/loss": 0.0028312649810686707 }, { "epoch": 1.5636509207365892, "grad_norm": 0.031551264226436615, "learning_rate": 1e-05, "loss": 2.86, "rewards/correctness_reward_func/mean": 0.75, "rewards/correctness_reward_func/std": 0.9759001135826111, "rewards/strict_format_reward_func/mean": 0.390625, "rewards/strict_format_reward_func/std": 0.2083333432674408, "step": 123, "time_profile/curr_logprobs_and_update": 0.261691846215399, "time_profile/old_logps": 33.64521975629032, "time_profile/ref_logps": 33.757645015604794, "time_profile/reward": 0.004097883589565754, "time_profile/text_rollout": 453.70345996133983, "train/gen_step": 7812.0, "und/advantages_abs_mean": 0.382568359375, "und/advantages_max": 1.1875, "und/advantages_mean": 0.0, "und/advantages_min": -2.1875, "und/advantages_std": 0.5867711305618286, "und/clip_ratio/high_mean": 0.0006388125475496054, "und/clip_ratio/low_mean": 0.0006393747644324321, "und/clip_ratio/region_mean": 0.0012781873083440587, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.76953125, "und/completion/min_length": 239.0, "und/frac_reward_zero_std": 0.125, "und/kl": 0.0320791233170894, "und/loss": 0.0024011894063278305, "und/prompt/max_length": 46.0, "und/prompt/mean_length": 44.5, "und/prompt/min_length": 43.0, "und/reward": 1.21875, "und/reward_std": 1.0983887910842896 }, { "epoch": 1.5764611689351482, "grad_norm": 0.03529511019587517, "learning_rate": 1e-05, "loss": 0.0066, "step": 124, "time_profile/curr_logprobs_and_update": 0.26139320741640404, "train/gen_step": 7876.0, "und/clip_ratio/high_mean": 0.0008959227816376369, "und/clip_ratio/low_mean": 0.00013536133337765932, "und/clip_ratio/region_mean": 0.0010312841150152963, "und/kl": 0.16470841343107168, "und/loss": 0.004291785090572375 }, { "epoch": 1.589271417133707, "grad_norm": 0.009518980048596859, "learning_rate": 1e-05, "loss": 0.0121, "rewards/correctness_reward_func/mean": 0.03125, "rewards/correctness_reward_func/std": 0.25, "rewards/strict_format_reward_func/mean": 0.0859375, "rewards/strict_format_reward_func/std": 0.19012710452079773, "step": 125, "time_profile/curr_logprobs_and_update": 0.26169546390883625, "time_profile/old_logps": 33.664723821915686, "time_profile/ref_logps": 33.749822195619345, "time_profile/reward": 0.004408887587487698, "time_profile/text_rollout": 505.58012927044183, "train/gen_step": 7940.0, "und/advantages_abs_mean": 0.13330078125, "und/advantages_max": 2.1875, "und/advantages_mean": 0.0, "und/advantages_min": -0.5625, "und/advantages_std": 0.26542437076568604, "und/clip_ratio/high_mean": 0.001534490605990868, "und/clip_ratio/low_mean": 0.0003314741697977297, "und/clip_ratio/region_mean": 0.0018659647794265766, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.87109375, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.4375, "und/kl": 0.08085775800282136, "und/loss": 0.009419288670414971, "und/prompt/max_length": 247.0, "und/prompt/mean_length": 204.5, "und/prompt/min_length": 162.0, "und/reward": 0.103515625, "und/reward_std": 0.29348325729370117 }, { "epoch": 1.6020816653322658, "grad_norm": 0.010913106612861156, "learning_rate": 1e-05, "loss": 0.0024, "step": 126, "time_profile/curr_logprobs_and_update": 0.2617497799510602, "train/gen_step": 8004.0, "und/clip_ratio/high_mean": 0.00018722502863965929, "und/clip_ratio/low_mean": 0.00034632114693522453, "und/clip_ratio/region_mean": 0.0005335461755748838, "und/kl": 0.06019564852613257, "und/loss": 0.002116124094982297 }, { "epoch": 1.6148919135308246, "grad_norm": 0.06803760677576065, "learning_rate": 1e-05, "loss": 0.0009, "rewards/correctness_reward_func/mean": 0.96875, "rewards/correctness_reward_func/std": 1.0074130296707153, "rewards/strict_format_reward_func/mean": 0.4375, "rewards/strict_format_reward_func/std": 0.1666666716337204, "step": 127, "time_profile/curr_logprobs_and_update": 0.2617532514850609, "time_profile/old_logps": 33.6751423580572, "time_profile/ref_logps": 33.763203986920416, "time_profile/reward": 0.004775381647050381, "time_profile/text_rollout": 471.52733379229903, "train/gen_step": 8068.0, "und/advantages_abs_mean": 0.903564453125, "und/advantages_max": 2.0625, "und/advantages_mean": 0.0, "und/advantages_min": -1.9375, "und/advantages_std": 1.0012528896331787, "und/clip_ratio/high_mean": 0.00015197079119388945, "und/clip_ratio/low_mean": 0.0012435419848770835, "und/clip_ratio/region_mean": 0.001395512772432994, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.599609375, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.015625, "und/kl": 0.0706367947277613, "und/loss": -0.004468285508210101, "und/prompt/max_length": 156.0, "und/prompt/mean_length": 102.5, "und/prompt/min_length": 49.0, "und/reward": 1.2705078125, "und/reward_std": 1.0752369165420532 }, { "epoch": 1.6277021617293834, "grad_norm": 0.07240945845842361, "learning_rate": 1e-05, "loss": 30.4224, "step": 128, "time_profile/curr_logprobs_and_update": 0.2616008282493567, "train/gen_step": 8132.0, "und/clip_ratio/high_mean": 0.0006445251565310173, "und/clip_ratio/low_mean": 0.00013259582192404196, "und/clip_ratio/region_mean": 0.0007771209748170804, "und/kl": 760.5635505191167, "und/loss": 243.34435985563323 }, { "epoch": 1.6405124099279424, "grad_norm": 0.06157054379582405, "learning_rate": 1e-05, "loss": 0.0101, "rewards/correctness_reward_func/mean": 0.15625, "rewards/correctness_reward_func/std": 0.5409794449806213, "rewards/strict_format_reward_func/mean": 0.2421875, "rewards/strict_format_reward_func/std": 0.25185325741767883, "step": 129, "time_profile/curr_logprobs_and_update": 0.26170297838689294, "time_profile/old_logps": 33.68404742050916, "time_profile/ref_logps": 33.77305310498923, "time_profile/reward": 0.004530956037342548, "time_profile/text_rollout": 473.9377162186429, "train/gen_step": 8196.0, "und/advantages_abs_mean": 0.3798828125, "und/advantages_max": 2.125, "und/advantages_mean": 0.0, "und/advantages_min": -1.0625, "und/advantages_std": 0.5802733898162842, "und/clip_ratio/high_mean": 0.0007628962412127294, "und/clip_ratio/low_mean": 0.0006278082109929528, "und/clip_ratio/region_mean": 0.0013907044594816398, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.736328125, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.015625, "und/kl": 0.1402410990849603, "und/loss": 0.008292756974697113, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 110.5, "und/prompt/min_length": 59.0, "und/reward": 0.353515625, "und/reward_std": 0.6494219303131104 }, { "epoch": 1.6533226581265013, "grad_norm": 0.03736671432852745, "learning_rate": 1e-05, "loss": 1.3177, "step": 130, "time_profile/curr_logprobs_and_update": 0.26144938450306654, "train/gen_step": 8260.0, "und/clip_ratio/high_mean": 0.000294210301944986, "und/clip_ratio/low_mean": 0.001453026579838479, "und/clip_ratio/region_mean": 0.001747236881783465, "und/kl": 32.937790766736725, "und/loss": 0.008295642333791875 }, { "epoch": 1.66613290632506, "grad_norm": 0.04435932636260986, "learning_rate": 1e-05, "loss": 15.2847, "rewards/correctness_reward_func/mean": 0.6875, "rewards/correctness_reward_func/std": 0.9574271440505981, "rewards/strict_format_reward_func/mean": 0.3203125, "rewards/strict_format_reward_func/std": 0.24180518090724945, "step": 131, "time_profile/curr_logprobs_and_update": 0.26175846286059823, "time_profile/old_logps": 33.65911100897938, "time_profile/ref_logps": 33.72500569187105, "time_profile/reward": 0.004387730732560158, "time_profile/text_rollout": 455.486419968307, "train/gen_step": 8324.0, "und/advantages_abs_mean": 0.45947265625, "und/advantages_max": 1.25, "und/advantages_mean": 0.0, "und/advantages_min": -2.1875, "und/advantages_std": 0.6482231616973877, "und/clip_ratio/high_mean": 0.0007765535956423264, "und/clip_ratio/low_mean": 0.0037881125645071734, "und/clip_ratio/region_mean": 0.004564666138321627, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.681640625, "und/completion/min_length": 252.0, "und/frac_reward_zero_std": 0.078125, "und/kl": 382.2992160025169, "und/loss": 2.063884374489703, "und/prompt/max_length": 55.0, "und/prompt/mean_length": 54.0, "und/prompt/min_length": 53.0, "und/reward": 1.0341796875, "und/reward_std": 1.1236227750778198 }, { "epoch": 1.6789431545236189, "grad_norm": 0.055891405791044235, "learning_rate": 1e-05, "loss": 0.0282, "step": 132, "time_profile/curr_logprobs_and_update": 0.2613213101722067, "train/gen_step": 8388.0, "und/clip_ratio/high_mean": 0.00035001601281692274, "und/clip_ratio/low_mean": 8.311169949593022e-05, "und/clip_ratio/region_mean": 0.0004331277195888106, "und/kl": 0.7055800975940656, "und/loss": 0.05676676309667528 }, { "epoch": 1.6917534027221777, "grad_norm": 0.04796978086233139, "learning_rate": 1e-05, "loss": 0.1329, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.1640625, "rewards/strict_format_reward_func/std": 0.2366211861371994, "step": 133, "time_profile/curr_logprobs_and_update": 0.2614839215821121, "time_profile/old_logps": 33.681789374910295, "time_profile/ref_logps": 33.78480255138129, "time_profile/reward": 0.004319341853260994, "time_profile/text_rollout": 491.34117817133665, "train/gen_step": 8452.0, "und/advantages_abs_mean": 0.18505859375, "und/advantages_max": 2.0, "und/advantages_mean": 0.0, "und/advantages_min": -0.5, "und/advantages_std": 0.2327723652124405, "und/clip_ratio/high_mean": 0.001052155526849674, "und/clip_ratio/low_mean": 0.0005224456072028261, "und/clip_ratio/region_mean": 0.0015746011340525001, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.810546875, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.078125, "und/kl": 3.207622427144088, "und/loss": 0.06492377794347703, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 162.0, "und/prompt/min_length": 162.0, "und/reward": 0.1484375, "und/reward_std": 0.2491423636674881 }, { "epoch": 1.7045636509207367, "grad_norm": 0.0103101572021842, "learning_rate": 1e-05, "loss": 619821.5625, "step": 134, "time_profile/curr_logprobs_and_update": 0.26158440396829974, "train/gen_step": 8516.0, "und/clip_ratio/high_mean": 0.0003344619144627359, "und/clip_ratio/low_mean": 0.003717986841365928, "und/clip_ratio/region_mean": 0.004052448763104621, "und/kl": 15495538.581971796, "und/loss": 442.40379129559733 }, { "epoch": 1.7173738991192953, "grad_norm": 0.03445025533437729, "learning_rate": 1e-05, "loss": 37837.5664, "rewards/correctness_reward_func/mean": 0.15625, "rewards/correctness_reward_func/std": 0.5409794449806213, "rewards/strict_format_reward_func/mean": 0.2421875, "rewards/strict_format_reward_func/std": 0.25185325741767883, "step": 135, "time_profile/curr_logprobs_and_update": 0.26179739559302106, "time_profile/old_logps": 33.6750593688339, "time_profile/ref_logps": 33.769202906638384, "time_profile/reward": 0.00483175553381443, "time_profile/text_rollout": 516.9765401165932, "train/gen_step": 8580.0, "und/advantages_abs_mean": 0.340087890625, "und/advantages_max": 1.8125, "und/advantages_mean": 0.0, "und/advantages_min": -1.1875, "und/advantages_std": 0.5872920155525208, "und/clip_ratio/high_mean": 0.0005857992764504161, "und/clip_ratio/low_mean": 0.004737480834592134, "und/clip_ratio/region_mean": 0.00532328011104255, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.693359375, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.5, "und/kl": 945939.298322209, "und/loss": 1624.2115884204977, "und/prompt/max_length": 313.0, "und/prompt/mean_length": 234.5, "und/prompt/min_length": 156.0, "und/reward": 0.474609375, "und/reward_std": 0.7803480625152588 }, { "epoch": 1.7301841473178543, "grad_norm": 0.041320931166410446, "learning_rate": 1e-05, "loss": 0.0137, "step": 136, "time_profile/curr_logprobs_and_update": 0.2617494677833747, "train/gen_step": 8644.0, "und/clip_ratio/high_mean": 0.00022294744121609256, "und/clip_ratio/low_mean": 0.00022158692809171043, "und/clip_ratio/region_mean": 0.000444534369307803, "und/kl": 0.3452570588560775, "und/loss": 0.010216924388146253 }, { "epoch": 1.742994395516413, "grad_norm": 0.028058085590600967, "learning_rate": 1e-05, "loss": 0.0285, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.0546875, "rewards/strict_format_reward_func/std": 0.15728822350502014, "step": 137, "time_profile/curr_logprobs_and_update": 0.2618869323923718, "time_profile/old_logps": 33.705197578296065, "time_profile/ref_logps": 33.796635060571134, "time_profile/reward": 0.00447302870452404, "time_profile/text_rollout": 519.2273662136868, "train/gen_step": 8708.0, "und/advantages_abs_mean": 0.087646484375, "und/advantages_max": 0.4375, "und/advantages_mean": 0.0, "und/advantages_min": -0.25, "und/advantages_std": 0.14817051589488983, "und/clip_ratio/high_mean": 0.0002926721135736443, "und/clip_ratio/low_mean": 0.00014689966701553203, "und/clip_ratio/region_mean": 0.00043957178058917634, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.908203125, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.546875, "und/kl": 0.6957836126966868, "und/loss": 0.15320786734139347, "und/prompt/max_length": 313.0, "und/prompt/mean_length": 237.5, "und/prompt/min_length": 162.0, "und/reward": 0.0673828125, "und/reward_std": 0.1709035038948059 }, { "epoch": 1.755804643714972, "grad_norm": 0.01070038229227066, "learning_rate": 1e-05, "loss": 73915.4844, "step": 138, "time_profile/curr_logprobs_and_update": 0.26177451500552706, "train/gen_step": 8772.0, "und/clip_ratio/high_mean": 0.00010704063606681302, "und/clip_ratio/low_mean": 0.0015135373505472671, "und/clip_ratio/region_mean": 0.0016205779829761013, "und/kl": 1847887.2981159356, "und/loss": 0.01662214804730411 }, { "epoch": 1.768614891913531, "grad_norm": 0.05411059781908989, "learning_rate": 1e-05, "loss": 179351.5938, "rewards/correctness_reward_func/mean": 0.90625, "rewards/correctness_reward_func/std": 1.003466248512268, "rewards/strict_format_reward_func/mean": 0.28125, "rewards/strict_format_reward_func/std": 0.25, "step": 139, "time_profile/curr_logprobs_and_update": 0.26185339997755364, "time_profile/old_logps": 33.67575552966446, "time_profile/ref_logps": 33.764580024406314, "time_profile/reward": 0.0031769955530762672, "time_profile/text_rollout": 491.6594071071595, "train/gen_step": 8836.0, "und/advantages_abs_mean": 1.008056640625, "und/advantages_max": 2.1875, "und/advantages_mean": 0.0, "und/advantages_min": -1.875, "und/advantages_std": 1.0957130193710327, "und/clip_ratio/high_mean": 0.00019178142974851653, "und/clip_ratio/low_mean": 0.0022219074999156874, "und/clip_ratio/region_mean": 0.002413688926026225, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.666015625, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.0, "und/kl": 4483789.968254499, "und/loss": 360.3619811698218, "und/prompt/max_length": 156.0, "und/prompt/mean_length": 156.0, "und/prompt/min_length": 156.0, "und/reward": 1.16015625, "und/reward_std": 1.161067008972168 }, { "epoch": 1.7814251401120895, "grad_norm": 0.1526426076889038, "learning_rate": 1e-05, "loss": 0.0064, "step": 140, "time_profile/curr_logprobs_and_update": 0.26170871233625803, "train/gen_step": 8900.0, "und/clip_ratio/high_mean": 0.0006244859760045074, "und/clip_ratio/low_mean": 0.000251247278356459, "und/clip_ratio/region_mean": 0.0008757332579989452, "und/kl": 0.16707535361638293, "und/loss": 0.0017974353395402431 }, { "epoch": 1.7942353883106485, "grad_norm": 0.04025937244296074, "learning_rate": 1e-05, "loss": 0.0154, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.1015625, "rewards/strict_format_reward_func/std": 0.20275264978408813, "step": 141, "time_profile/curr_logprobs_and_update": 0.26171166240237653, "time_profile/old_logps": 33.693387147039175, "time_profile/ref_logps": 33.75184701196849, "time_profile/reward": 0.00441189669072628, "time_profile/text_rollout": 518.2822579685599, "train/gen_step": 8964.0, "und/advantages_abs_mean": 0.106201171875, "und/advantages_max": 2.125, "und/advantages_mean": 0.0, "und/advantages_min": -0.375, "und/advantages_std": 0.18686699867248535, "und/clip_ratio/high_mean": 0.0013163000876375008, "und/clip_ratio/low_mean": 0.0009350144391646609, "und/clip_ratio/region_mean": 0.002251314519526204, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.88671875, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.515625, "und/kl": 0.1287980208871886, "und/loss": 0.028064919478310912, "und/prompt/max_length": 313.0, "und/prompt/mean_length": 237.5, "und/prompt/min_length": 162.0, "und/reward": 0.0888671875, "und/reward_std": 0.21538907289505005 }, { "epoch": 1.8070456365092074, "grad_norm": 0.010626863688230515, "learning_rate": 1e-05, "loss": 260079.9062, "step": 142, "time_profile/curr_logprobs_and_update": 0.2617065573576838, "train/gen_step": 9028.0, "und/clip_ratio/high_mean": 5.5803571740398183e-05, "und/clip_ratio/low_mean": 0.0006520278329844587, "und/clip_ratio/region_mean": 0.0007078314047248568, "und/kl": 6501997.801571596, "und/loss": 2068612.8666643738 }, { "epoch": 1.8198558847077662, "grad_norm": 0.0026135630905628204, "learning_rate": 1e-05, "loss": 8.5116, "rewards/correctness_reward_func/mean": 1.0, "rewards/correctness_reward_func/std": 1.0079052448272705, "rewards/strict_format_reward_func/mean": 0.25, "rewards/strict_format_reward_func/std": 0.2519763112068176, "step": 143, "time_profile/curr_logprobs_and_update": 0.26167610530683305, "time_profile/old_logps": 33.65058270841837, "time_profile/ref_logps": 33.7399518340826, "time_profile/reward": 0.004332819953560829, "time_profile/text_rollout": 490.11245891079307, "train/gen_step": 9092.0, "und/advantages_abs_mean": 0.08447265625, "und/advantages_max": 2.1875, "und/advantages_mean": 0.0, "und/advantages_min": -2.1875, "und/advantages_std": 0.2977820634841919, "und/clip_ratio/high_mean": 0.00011625744446064346, "und/clip_ratio/low_mean": 0.0014596742403227836, "und/clip_ratio/region_mean": 0.0015759316884214059, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.814453125, "und/completion/min_length": 215.0, "und/frac_reward_zero_std": 0.765625, "und/kl": 212.7892216272594, "und/loss": 0.10032683075525028, "und/prompt/max_length": 247.0, "und/prompt/mean_length": 145.5, "und/prompt/min_length": 44.0, "und/reward": 1.240234375, "und/reward_std": 1.2425521612167358 }, { "epoch": 1.8326661329063252, "grad_norm": 0.00504687987267971, "learning_rate": 1e-05, "loss": 0.0149, "step": 144, "time_profile/curr_logprobs_and_update": 0.2615732659032801, "train/gen_step": 9156.0, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 4.882812572759576e-05, "und/clip_ratio/region_mean": 4.882812572759576e-05, "und/kl": 0.372995498473756, "und/loss": 0.006929575064759774 }, { "epoch": 1.8454763811048838, "grad_norm": 0.052012741565704346, "learning_rate": 1e-05, "loss": 0.0058, "rewards/correctness_reward_func/mean": 0.5, "rewards/correctness_reward_func/std": 0.8728715777397156, "rewards/strict_format_reward_func/mean": 0.2265625, "rewards/strict_format_reward_func/std": 0.250866562128067, "step": 145, "time_profile/curr_logprobs_and_update": 0.26171143807005137, "time_profile/old_logps": 33.68571895174682, "time_profile/ref_logps": 33.76929687988013, "time_profile/reward": 0.004742258228361607, "time_profile/text_rollout": 489.8518072050065, "train/gen_step": 9220.0, "und/advantages_abs_mean": 0.578369140625, "und/advantages_max": 2.0625, "und/advantages_mean": 0.0, "und/advantages_min": -1.75, "und/advantages_std": 0.7647761702537537, "und/clip_ratio/high_mean": 0.00015467782941414043, "und/clip_ratio/low_mean": 4.542151145869866e-05, "und/clip_ratio/region_mean": 0.0002000993408728391, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.771484375, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.015625, "und/kl": 0.46287906426005065, "und/loss": -0.01604961227064905, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 159.0, "und/prompt/min_length": 156.0, "und/reward": 0.703125, "und/reward_std": 0.9880647659301758 }, { "epoch": 1.8582866293034428, "grad_norm": 0.0647926777601242, "learning_rate": 1e-05, "loss": 13804.5205, "step": 146, "time_profile/curr_logprobs_and_update": 0.26155846660549287, "train/gen_step": 9284.0, "und/clip_ratio/high_mean": 0.00046555879816878587, "und/clip_ratio/low_mean": 0.0007697123000980355, "und/clip_ratio/region_mean": 0.0012352710982668214, "und/kl": 345113.0145757744, "und/loss": 1.7385565766200628 }, { "epoch": 1.8710968775020016, "grad_norm": 6.310885906219482, "learning_rate": 1e-05, "loss": 148005.3906, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.09375, "rewards/strict_format_reward_func/std": 0.19669894874095917, "step": 147, "time_profile/curr_logprobs_and_update": 0.2616866462340113, "time_profile/old_logps": 33.69251477625221, "time_profile/ref_logps": 33.79237876459956, "time_profile/reward": 0.004402017220854759, "time_profile/text_rollout": 491.3923704344779, "train/gen_step": 9348.0, "und/advantages_abs_mean": 0.169677734375, "und/advantages_max": 0.4375, "und/advantages_mean": 0.0, "und/advantages_min": -0.25, "und/advantages_std": 0.20616121590137482, "und/clip_ratio/high_mean": 0.0011721192167897243, "und/clip_ratio/low_mean": 0.001822430691390764, "und/clip_ratio/region_mean": 0.002994549911818467, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.83984375, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.078125, "und/kl": 3700134.6438976433, "und/loss": 27.951937119380318, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 162.0, "und/prompt/min_length": 162.0, "und/reward": 0.1298828125, "und/reward_std": 0.2194673866033554 }, { "epoch": 1.8839071257005604, "grad_norm": 220.56588745117188, "learning_rate": 1e-05, "loss": 6.8814, "step": 148, "time_profile/curr_logprobs_and_update": 0.26165977913478855, "train/gen_step": 9412.0, "und/clip_ratio/high_mean": 0.0023689054542046506, "und/clip_ratio/low_mean": 0.0010092035663546994, "und/clip_ratio/region_mean": 0.003378109024197329, "und/kl": 68.11535315206856, "und/loss": 2.0457846918769746 }, { "epoch": 1.8967173738991194, "grad_norm": 1.7173937559127808, "learning_rate": 1e-05, "loss": 9899996.0, "rewards/correctness_reward_func/mean": 0.5625, "rewards/correctness_reward_func/std": 0.9063270092010498, "rewards/strict_format_reward_func/mean": 0.265625, "rewards/strict_format_reward_func/std": 0.2514837086200714, "step": 149, "time_profile/curr_logprobs_and_update": 0.2616911970835645, "time_profile/old_logps": 33.67640256509185, "time_profile/ref_logps": 33.75797679182142, "time_profile/reward": 0.004289323464035988, "time_profile/text_rollout": 474.0713466331363, "train/gen_step": 9476.0, "und/advantages_abs_mean": 0.607421875, "und/advantages_max": 2.1875, "und/advantages_mean": 0.0, "und/advantages_min": -2.1875, "und/advantages_std": 0.7971558570861816, "und/clip_ratio/high_mean": 0.0029124355169187766, "und/clip_ratio/low_mean": 0.0017562373941473197, "und/clip_ratio/region_mean": 0.004668672914704075, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.71875, "und/completion/min_length": 251.0, "und/frac_reward_zero_std": 0.015625, "und/kl": 247499896.9961213, "und/loss": 437175.3625415751, "und/prompt/max_length": 156.0, "und/prompt/mean_length": 105.0, "und/prompt/min_length": 54.0, "und/reward": 0.8623046875, "und/reward_std": 1.0626578330993652 }, { "epoch": 1.909527622097678, "grad_norm": 0.011920370161533356, "learning_rate": 1e-05, "loss": 0.0134, "step": 150, "time_profile/curr_logprobs_and_update": 0.26148566768097226, "train/gen_step": 9540.0, "und/clip_ratio/high_mean": 0.000891603427589871, "und/clip_ratio/low_mean": 0.00019111833535134792, "und/clip_ratio/region_mean": 0.001082721762941219, "und/kl": 0.3309498258458916, "und/loss": 0.0035182901192456484 }, { "epoch": 1.922337870296237, "grad_norm": 0.04238224774599075, "learning_rate": 1e-05, "loss": -0.0036, "rewards/correctness_reward_func/mean": 0.5625, "rewards/correctness_reward_func/std": 0.9063270092010498, "rewards/strict_format_reward_func/mean": 0.3671875, "rewards/strict_format_reward_func/std": 0.22257846593856812, "step": 151, "time_profile/curr_logprobs_and_update": 0.2615245400520507, "time_profile/old_logps": 33.659803338348866, "time_profile/ref_logps": 33.75399888958782, "time_profile/reward": 0.00458063930273056, "time_profile/text_rollout": 471.09525199607015, "train/gen_step": 9604.0, "und/advantages_abs_mean": 0.615234375, "und/advantages_max": 2.0625, "und/advantages_mean": 0.0, "und/advantages_min": -1.5, "und/advantages_std": 0.7951586842536926, "und/clip_ratio/high_mean": 0.0020844707141804975, "und/clip_ratio/low_mean": 0.0001922483897942584, "und/clip_ratio/region_mean": 0.002276719100336777, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.716796875, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.015625, "und/kl": 0.05341187393059954, "und/loss": 0.009631790686398745, "und/prompt/max_length": 156.0, "und/prompt/mean_length": 99.5, "und/prompt/min_length": 43.0, "und/reward": 0.8447265625, "und/reward_std": 0.9140197038650513 }, { "epoch": 1.9351481184947958, "grad_norm": 0.06001448258757591, "learning_rate": 1e-05, "loss": 0.0095, "step": 152, "time_profile/curr_logprobs_and_update": 0.261558934187633, "train/gen_step": 9668.0, "und/clip_ratio/high_mean": 0.0005441326029540505, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0005441326029540505, "und/kl": 0.2372699785628356, "und/loss": 0.0034325564286064036 }, { "epoch": 1.9479583666933546, "grad_norm": 0.026463812217116356, "learning_rate": 1e-05, "loss": 0.0137, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.1875, "rewards/strict_format_reward_func/std": 0.24397502839565277, "step": 153, "time_profile/curr_logprobs_and_update": 0.2616782131517539, "time_profile/old_logps": 33.68901115562767, "time_profile/ref_logps": 33.789112674072385, "time_profile/reward": 0.004856881685554981, "time_profile/text_rollout": 476.91363042313606, "train/gen_step": 9732.0, "und/advantages_abs_mean": 0.197509765625, "und/advantages_max": 0.4375, "und/advantages_mean": 0.0, "und/advantages_min": -0.375, "und/advantages_std": 0.22242768108844757, "und/clip_ratio/high_mean": 0.0009648025552451145, "und/clip_ratio/low_mean": 6.734913768013939e-05, "und/clip_ratio/region_mean": 0.0010321516929252539, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.77734375, "und/completion/min_length": 252.0, "und/frac_reward_zero_std": 0.03125, "und/kl": 0.22904289467260242, "und/loss": -0.018844104985873855, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 111.5, "und/prompt/min_length": 61.0, "und/reward": 0.1689453125, "und/reward_std": 0.2367268204689026 }, { "epoch": 1.9607686148919137, "grad_norm": 152.77015686035156, "learning_rate": 1e-05, "loss": 0.5685, "step": 154, "time_profile/curr_logprobs_and_update": 0.26678641652688384, "train/gen_step": 9796.0, "und/clip_ratio/high_mean": 0.0005597944764303975, "und/clip_ratio/low_mean": 0.00025797137641347945, "und/clip_ratio/region_mean": 0.0008177658528438769, "und/kl": 14.211894012492849, "und/loss": 0.004249518504366279 }, { "epoch": 1.9735788630904723, "grad_norm": 0.010165955871343613, "learning_rate": 1e-05, "loss": 0.015, "rewards/correctness_reward_func/mean": 0.15625, "rewards/correctness_reward_func/std": 0.5409794449806213, "rewards/strict_format_reward_func/mean": 0.3359375, "rewards/strict_format_reward_func/std": 0.2366211861371994, "step": 155, "time_profile/curr_logprobs_and_update": 0.2616322033718461, "time_profile/old_logps": 33.655117710120976, "time_profile/ref_logps": 33.740064659155905, "time_profile/reward": 0.004281140863895416, "time_profile/text_rollout": 472.37845607008785, "train/gen_step": 9860.0, "und/advantages_abs_mean": 0.363037109375, "und/advantages_max": 1.8125, "und/advantages_mean": 0.0, "und/advantages_min": -1.5, "und/advantages_std": 0.5413435697555542, "und/clip_ratio/high_mean": 0.0021703395432268735, "und/clip_ratio/low_mean": 0.0006125769177742768, "und/clip_ratio/region_mean": 0.002782916464639129, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.7578125, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.109375, "und/kl": 0.2890739535650937, "und/loss": 0.0022308455081656575, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 103.0, "und/prompt/min_length": 44.0, "und/reward": 0.53515625, "und/reward_std": 0.6929434537887573 }, { "epoch": 1.9863891112890313, "grad_norm": 0.033500850200653076, "learning_rate": 1e-05, "loss": 0.0053, "step": 156, "time_profile/curr_logprobs_and_update": 0.2615450029115891, "train/gen_step": 9924.0, "und/clip_ratio/high_mean": 0.0014465084786934312, "und/clip_ratio/low_mean": 0.00015304287444450893, "und/clip_ratio/region_mean": 0.0015995513422240037, "und/kl": 0.12355754713644274, "und/loss": 0.017674150729817484 }, { "epoch": 1.99919935948759, "grad_norm": 0.011605536565184593, "learning_rate": 1e-05, "loss": 0.0113, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.09375, "rewards/strict_format_reward_func/std": 0.19669894874095917, "step": 157, "time_profile/curr_logprobs_and_update": 0.26163526308664586, "time_profile/old_logps": 33.68655322957784, "time_profile/ref_logps": 33.78236538451165, "time_profile/reward": 0.0044804466888308525, "time_profile/text_rollout": 518.2296011028811, "train/gen_step": 9988.0, "und/advantages_abs_mean": 0.12353515625, "und/advantages_max": 2.1875, "und/advantages_mean": 0.0, "und/advantages_min": -0.375, "und/advantages_std": 0.19876083731651306, "und/clip_ratio/high_mean": 0.001845829377998598, "und/clip_ratio/low_mean": 0.000797167816926958, "und/clip_ratio/region_mean": 0.002642997173097683, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.859375, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.453125, "und/kl": 0.2580548677360639, "und/loss": 0.016669023714030118, "und/prompt/max_length": 313.0, "und/prompt/mean_length": 237.5, "und/prompt/min_length": 162.0, "und/reward": 0.109375, "und/reward_std": 0.22933153808116913 }, { "epoch": 2.0, "grad_norm": 0.008261620998382568, "learning_rate": 1e-05, "loss": -0.0022, "step": 158, "time_profile/curr_logprobs_and_update": 0.2615143822040409, "train/gen_step": 9992.0, "und/clip_ratio/high_mean": 0.0008680555620230734, "und/clip_ratio/low_mean": 0.0017684109043329954, "und/clip_ratio/region_mean": 0.002636466466356069, "und/kl": 0.0970268496312201, "und/loss": -0.06052475795149803 }, { "epoch": 2.012810248198559, "grad_norm": 0.007182754576206207, "learning_rate": 1e-05, "loss": 0.0027, "step": 159, "time_profile/curr_logprobs_and_update": 0.2617839087470202, "train/gen_step": 10056.0, "und/clip_ratio/high_mean": 0.0009972724728868343, "und/clip_ratio/low_mean": 0.00033537117633386515, "und/clip_ratio/region_mean": 0.0013326436528586783, "und/kl": 0.06421022860740777, "und/loss": 0.0018333349892145634 }, { "epoch": 2.0256204963971176, "grad_norm": 0.00574113754555583, "learning_rate": 1e-05, "loss": 0.0029, "rewards/correctness_reward_func/mean": 0.5625, "rewards/correctness_reward_func/std": 0.9063270092010498, "rewards/strict_format_reward_func/mean": 0.34375, "rewards/strict_format_reward_func/std": 0.233588308095932, "step": 160, "time_profile/curr_logprobs_and_update": 0.2617837716679787, "time_profile/old_logps": 33.679151609539986, "time_profile/ref_logps": 33.74794678296894, "time_profile/reward": 0.004701966419816017, "time_profile/text_rollout": 472.4926246367395, "train/gen_step": 10120.0, "und/advantages_abs_mean": 0.545654296875, "und/advantages_max": 1.625, "und/advantages_mean": 0.0, "und/advantages_min": -2.1875, "und/advantages_std": 0.6981857419013977, "und/clip_ratio/high_mean": 0.0016045431657403242, "und/clip_ratio/low_mean": 0.00043878793803742155, "und/clip_ratio/region_mean": 0.0020433310928638093, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.75, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.0, "und/kl": 0.07028049726795871, "und/loss": 0.007069828358226005, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 103.5, "und/prompt/min_length": 45.0, "und/reward": 0.9853515625, "und/reward_std": 1.0728275775909424 }, { "epoch": 2.0384307445956766, "grad_norm": 0.08203817158937454, "learning_rate": 1e-05, "loss": 0.0029, "step": 161, "time_profile/curr_logprobs_and_update": 0.2615642463497352, "train/gen_step": 10184.0, "und/clip_ratio/high_mean": 0.00023133041395340115, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.00023133041395340115, "und/kl": 0.06791361824434716, "und/loss": 0.0056212409399449825 }, { "epoch": 2.051240992794235, "grad_norm": 0.09558697044849396, "learning_rate": 1e-05, "loss": 0.0037, "rewards/correctness_reward_func/mean": 0.3125, "rewards/correctness_reward_func/std": 0.7319250702857971, "rewards/strict_format_reward_func/mean": 0.328125, "rewards/strict_format_reward_func/std": 0.23935678601264954, "step": 162, "time_profile/curr_logprobs_and_update": 0.26180726967868395, "time_profile/old_logps": 33.68221877235919, "time_profile/ref_logps": 33.787430736236274, "time_profile/reward": 0.004500852897763252, "time_profile/text_rollout": 472.37788531929255, "train/gen_step": 10248.0, "und/advantages_abs_mean": 0.521484375, "und/advantages_max": 2.0, "und/advantages_mean": 0.0, "und/advantages_min": -1.625, "und/advantages_std": 0.6933073997497559, "und/clip_ratio/high_mean": 0.0011127722791570704, "und/clip_ratio/low_mean": 0.001079595233022701, "und/clip_ratio/region_mean": 0.0021923675085417926, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.630859375, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.0, "und/kl": 0.10435478831641376, "und/loss": 0.00502297020284459, "und/prompt/max_length": 156.0, "und/prompt/mean_length": 105.5, "und/prompt/min_length": 55.0, "und/reward": 0.6328125, "und/reward_std": 0.8416374921798706 }, { "epoch": 2.0640512409927942, "grad_norm": 0.04183354601264, "learning_rate": 1e-05, "loss": 0.0066, "step": 163, "time_profile/curr_logprobs_and_update": 0.26139140441955533, "train/gen_step": 10312.0, "und/clip_ratio/high_mean": 0.00019865745343849994, "und/clip_ratio/low_mean": 0.00014263017146731727, "und/clip_ratio/region_mean": 0.0003412876249058172, "und/kl": 0.16455070252413861, "und/loss": 0.002166528080124408 }, { "epoch": 2.0768614891913533, "grad_norm": 0.009151386097073555, "learning_rate": 1e-05, "loss": 0.0025, "rewards/correctness_reward_func/mean": 0.0625, "rewards/correctness_reward_func/std": 0.35073620080947876, "rewards/strict_format_reward_func/mean": 0.34375, "rewards/strict_format_reward_func/std": 0.233588308095932, "step": 164, "time_profile/curr_logprobs_and_update": 0.26157606378546916, "time_profile/old_logps": 33.671810798346996, "time_profile/ref_logps": 33.7788968719542, "time_profile/reward": 0.004629682749509811, "time_profile/text_rollout": 472.72225496545434, "train/gen_step": 10376.0, "und/advantages_abs_mean": 0.18994140625, "und/advantages_max": 1.75, "und/advantages_mean": 0.0, "und/advantages_min": -0.5, "und/advantages_std": 0.3282596468925476, "und/clip_ratio/high_mean": 0.0003861400473397225, "und/clip_ratio/low_mean": 0.00010074145757243969, "und/clip_ratio/region_mean": 0.0004868815049121622, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.779296875, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.3125, "und/kl": 0.06336885890777921, "und/loss": 0.0024055056273937225, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 102.5, "und/prompt/min_length": 43.0, "und/reward": 0.392578125, "und/reward_std": 0.39958831667900085 }, { "epoch": 2.089671737389912, "grad_norm": 0.0061566890217363834, "learning_rate": 1e-05, "loss": 0.0062, "step": 165, "time_profile/curr_logprobs_and_update": 0.26149553031427786, "train/gen_step": 10440.0, "und/clip_ratio/high_mean": 0.0001600477480678819, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0001600477480678819, "und/kl": 0.154379196363152, "und/loss": 0.0021597322743218683 }, { "epoch": 2.102481985588471, "grad_norm": 0.06113005802035332, "learning_rate": 1e-05, "loss": 0.0048, "rewards/correctness_reward_func/mean": 0.78125, "rewards/correctness_reward_func/std": 0.983494758605957, "rewards/strict_format_reward_func/mean": 0.375, "rewards/strict_format_reward_func/std": 0.2182178944349289, "step": 166, "time_profile/curr_logprobs_and_update": 0.2617073591245571, "time_profile/old_logps": 33.64431105554104, "time_profile/ref_logps": 33.74050881713629, "time_profile/reward": 0.004514393396675587, "time_profile/text_rollout": 454.8430243739858, "train/gen_step": 10504.0, "und/advantages_abs_mean": 0.470703125, "und/advantages_max": 1.5, "und/advantages_mean": 0.0, "und/advantages_min": -1.9375, "und/advantages_std": 0.6300917863845825, "und/clip_ratio/high_mean": 0.0004267864424036816, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0004267864424036816, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.6953125, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.046875, "und/kl": 0.11857564687670674, "und/loss": 0.003597188484036451, "und/prompt/max_length": 50.0, "und/prompt/mean_length": 50.0, "und/prompt/min_length": 50.0, "und/reward": 1.08984375, "und/reward_std": 1.0457885265350342 }, { "epoch": 2.1152922337870295, "grad_norm": 0.03352009132504463, "learning_rate": 1e-05, "loss": 0.006, "step": 167, "time_profile/curr_logprobs_and_update": 0.26128566317493096, "train/gen_step": 10568.0, "und/clip_ratio/high_mean": 0.0015908839777694084, "und/clip_ratio/low_mean": 4.542151145869866e-05, "und/clip_ratio/region_mean": 0.001636305489228107, "und/kl": 0.11126315701403655, "und/loss": 0.004314804737987288 }, { "epoch": 2.1281024819855885, "grad_norm": 0.02977151609957218, "learning_rate": 1e-05, "loss": 0.0109, "rewards/correctness_reward_func/mean": 0.03125, "rewards/correctness_reward_func/std": 0.25, "rewards/strict_format_reward_func/mean": 0.25, "rewards/strict_format_reward_func/std": 0.2519763112068176, "step": 168, "time_profile/curr_logprobs_and_update": 0.2614454288268462, "time_profile/old_logps": 33.630763033404946, "time_profile/ref_logps": 33.74494473170489, "time_profile/reward": 0.004755772650241852, "time_profile/text_rollout": 487.7817754801363, "train/gen_step": 10632.0, "und/advantages_abs_mean": 0.058349609375, "und/advantages_max": 2.125, "und/advantages_mean": 0.0, "und/advantages_min": -0.5, "und/advantages_std": 0.20196563005447388, "und/clip_ratio/high_mean": 0.0013399315866990946, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0013399315866990946, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.71484375, "und/completion/min_length": 221.0, "und/frac_reward_zero_std": 0.703125, "und/kl": 0.047248675022274256, "und/loss": 0.01837195276038983, "und/prompt/max_length": 247.0, "und/prompt/mean_length": 149.0, "und/prompt/min_length": 51.0, "und/reward": 0.2822265625, "und/reward_std": 0.30478331446647644 }, { "epoch": 2.1409127301841475, "grad_norm": 0.008049830794334412, "learning_rate": 1e-05, "loss": 0.0037, "step": 169, "time_profile/curr_logprobs_and_update": 0.26160103428992443, "train/gen_step": 10696.0, "und/clip_ratio/high_mean": 0.00010557432688074186, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.00010557432688074186, "und/kl": 0.09378908702638, "und/loss": 0.002309132060418051 }, { "epoch": 2.153722978382706, "grad_norm": 0.03837805986404419, "learning_rate": 1e-05, "loss": 0.0025, "rewards/correctness_reward_func/mean": 0.71875, "rewards/correctness_reward_func/std": 0.9672207236289978, "rewards/strict_format_reward_func/mean": 0.40625, "rewards/strict_format_reward_func/std": 0.19669894874095917, "step": 170, "time_profile/curr_logprobs_and_update": 0.2617557874182239, "time_profile/old_logps": 33.67024072445929, "time_profile/ref_logps": 33.74819596670568, "time_profile/reward": 0.004233397543430328, "time_profile/text_rollout": 488.64935347717255, "train/gen_step": 10760.0, "und/advantages_abs_mean": 0.906982421875, "und/advantages_max": 2.0, "und/advantages_mean": 0.0, "und/advantages_min": -1.9375, "und/advantages_std": 1.0159564018249512, "und/clip_ratio/high_mean": 0.00015850567069719546, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.00015850567069719546, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.59375, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.0, "und/kl": 0.0635057357721962, "und/loss": 0.00192016769960901, "und/prompt/max_length": 156.0, "und/prompt/mean_length": 156.0, "und/prompt/min_length": 156.0, "und/reward": 1.123046875, "und/reward_std": 1.0842657089233398 }, { "epoch": 2.166533226581265, "grad_norm": 0.04740050062537193, "learning_rate": 1e-05, "loss": 0.0057, "step": 171, "time_profile/curr_logprobs_and_update": 0.2614927099493798, "train/gen_step": 10824.0, "und/clip_ratio/high_mean": 0.00019523592709447257, "und/clip_ratio/low_mean": 5.278716344037093e-05, "und/clip_ratio/region_mean": 0.0002480230905348435, "und/kl": 0.14967676892410964, "und/loss": 0.002013998571783304 }, { "epoch": 2.1793434747798237, "grad_norm": 0.03162504360079765, "learning_rate": 1e-05, "loss": 0.0032, "rewards/correctness_reward_func/mean": 0.09375, "rewards/correctness_reward_func/std": 0.42608407139778137, "rewards/strict_format_reward_func/mean": 0.2734375, "rewards/strict_format_reward_func/std": 0.250866562128067, "step": 172, "time_profile/curr_logprobs_and_update": 0.2618536733352812, "time_profile/old_logps": 33.67922029271722, "time_profile/ref_logps": 33.76980642415583, "time_profile/reward": 0.004333090037107468, "time_profile/text_rollout": 473.33523644600064, "train/gen_step": 10888.0, "und/advantages_abs_mean": 0.21728515625, "und/advantages_max": 2.0, "und/advantages_mean": 0.0, "und/advantages_min": -0.6875, "und/advantages_std": 0.2858365774154663, "und/clip_ratio/high_mean": 0.0005348777740437072, "und/clip_ratio/low_mean": 0.0002211474893556442, "und/clip_ratio/region_mean": 0.0007560252670373302, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.728515625, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.046875, "und/kl": 0.08075993752572685, "und/loss": 0.00289958156645298, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 107.5, "und/prompt/min_length": 53.0, "und/reward": 0.27734375, "und/reward_std": 0.31782403588294983 }, { "epoch": 2.1921537229783827, "grad_norm": 0.011727745644748211, "learning_rate": 1e-05, "loss": 0.0019, "step": 173, "time_profile/curr_logprobs_and_update": 0.2616145513165975, "train/gen_step": 10952.0, "und/clip_ratio/high_mean": 6.103515625e-05, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 6.103515625e-05, "und/kl": 0.04787289367232006, "und/loss": 0.001894820132292807 }, { "epoch": 2.2049639711769418, "grad_norm": 0.008552194572985172, "learning_rate": 1e-05, "loss": 0.0021, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.2890625, "rewards/strict_format_reward_func/std": 0.24888142943382263, "step": 174, "time_profile/curr_logprobs_and_update": 0.26172452403989155, "time_profile/old_logps": 33.6549604870379, "time_profile/ref_logps": 33.75101667176932, "time_profile/reward": 0.004562413319945335, "time_profile/text_rollout": 455.8837287509814, "train/gen_step": 11016.0, "und/advantages_abs_mean": 0.2109375, "und/advantages_max": 0.4375, "und/advantages_mean": 0.0, "und/advantages_min": -0.4375, "und/advantages_std": 0.22986425459384918, "und/clip_ratio/high_mean": 0.00026825524037121795, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.00026825524037121795, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.73046875, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.015625, "und/kl": 0.054617739195236936, "und/loss": 0.0014219468575902283, "und/prompt/max_length": 58.0, "und/prompt/mean_length": 55.5, "und/prompt/min_length": 53.0, "und/reward": 0.28515625, "und/reward_std": 0.24775780737400055 }, { "epoch": 2.2177742193755003, "grad_norm": 0.011380493640899658, "learning_rate": 1e-05, "loss": 0.0021, "step": 175, "time_profile/curr_logprobs_and_update": 0.2613375957298558, "train/gen_step": 11080.0, "und/clip_ratio/high_mean": 0.0009292362919950392, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0009292362919950392, "und/kl": 0.053230694044032134, "und/loss": 0.001982355781365186 }, { "epoch": 2.2305844675740594, "grad_norm": 0.03694207966327667, "learning_rate": 1e-05, "loss": 0.0018, "rewards/correctness_reward_func/mean": 0.625, "rewards/correctness_reward_func/std": 0.934353232383728, "rewards/strict_format_reward_func/mean": 0.1640625, "rewards/strict_format_reward_func/std": 0.2366211861371994, "step": 176, "time_profile/curr_logprobs_and_update": 0.2614654813223751, "time_profile/old_logps": 33.645203595981, "time_profile/ref_logps": 33.76335656829178, "time_profile/reward": 0.004439213313162327, "time_profile/text_rollout": 517.072899880819, "train/gen_step": 11144.0, "und/advantages_abs_mean": 0.5068359375, "und/advantages_max": 2.1875, "und/advantages_mean": 0.0, "und/advantages_min": -2.1875, "und/advantages_std": 0.777386486530304, "und/clip_ratio/high_mean": 0.001711124616122106, "und/clip_ratio/low_mean": 9.192448851536028e-05, "und/clip_ratio/region_mean": 0.0018030491046374664, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.78515625, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.34375, "und/kl": 0.04419090352894273, "und/loss": 0.001942948205396533, "und/prompt/max_length": 313.0, "und/prompt/mean_length": 234.5, "und/prompt/min_length": 156.0, "und/reward": 0.90234375, "und/reward_std": 1.177177906036377 }, { "epoch": 2.243394715772618, "grad_norm": 0.04087910056114197, "learning_rate": 1e-05, "loss": 0.025, "step": 177, "time_profile/curr_logprobs_and_update": 0.2616091024246998, "train/gen_step": 11208.0, "und/clip_ratio/high_mean": 0.0001443542423658073, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0001443542423658073, "und/kl": 0.6302279950177763, "und/loss": 0.0017156366041035653 }, { "epoch": 2.256204963971177, "grad_norm": 0.06869915127754211, "learning_rate": 1e-05, "loss": 0.0761, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.1015625, "rewards/strict_format_reward_func/std": 0.20275264978408813, "step": 178, "time_profile/curr_logprobs_and_update": 0.26177879216265865, "time_profile/old_logps": 33.71037639584392, "time_profile/ref_logps": 33.78417940158397, "time_profile/reward": 0.004494715481996536, "time_profile/text_rollout": 518.1308631561697, "train/gen_step": 11272.0, "und/advantages_abs_mean": 0.1259765625, "und/advantages_max": 2.1875, "und/advantages_mean": 0.0, "und/advantages_min": -0.5625, "und/advantages_std": 0.23105835914611816, "und/clip_ratio/high_mean": 0.0004932920819555875, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0004932920819555875, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.859375, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.484375, "und/kl": 1.9037787910492625, "und/loss": 0.41365190602118673, "und/prompt/max_length": 313.0, "und/prompt/mean_length": 237.5, "und/prompt/min_length": 162.0, "und/reward": 0.1162109375, "und/reward_std": 0.26848655939102173 }, { "epoch": 2.2690152121697356, "grad_norm": 0.00810084119439125, "learning_rate": 1e-05, "loss": 0.0022, "step": 179, "time_profile/curr_logprobs_and_update": 0.2615918532537762, "train/gen_step": 11336.0, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/kl": 0.054657230590237305, "und/loss": 0.0015159287313508685 }, { "epoch": 2.2818254603682946, "grad_norm": 0.010132861323654652, "learning_rate": 1e-05, "loss": 0.0029, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.234375, "rewards/strict_format_reward_func/std": 0.2514837086200714, "step": 180, "time_profile/curr_logprobs_and_update": 0.2618380212079501, "time_profile/old_logps": 33.66110870428383, "time_profile/ref_logps": 33.76540139503777, "time_profile/reward": 0.004269592463970184, "time_profile/text_rollout": 473.37367711402476, "train/gen_step": 11400.0, "und/advantages_abs_mean": 0.229736328125, "und/advantages_max": 2.0, "und/advantages_mean": 0.0, "und/advantages_min": -0.5625, "und/advantages_std": 0.27835747599601746, "und/clip_ratio/high_mean": 0.000102588386653224, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.000102588386653224, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.732421875, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.03125, "und/kl": 0.07371564902132377, "und/loss": 0.0017190172418395377, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 108.0, "und/prompt/min_length": 54.0, "und/reward": 0.2607421875, "und/reward_std": 0.29655858874320984 }, { "epoch": 2.2946357085668536, "grad_norm": 0.008120850659906864, "learning_rate": 1e-05, "loss": 0.0025, "step": 181, "time_profile/curr_logprobs_and_update": 0.26148502981232014, "train/gen_step": 11464.0, "und/clip_ratio/high_mean": 0.0003160519918310456, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0003160519918310456, "und/kl": 0.06285516830394045, "und/loss": 0.0015067223123139684 }, { "epoch": 2.307445956765412, "grad_norm": 0.009181732311844826, "learning_rate": 1e-05, "loss": 0.0021, "rewards/correctness_reward_func/mean": 0.125, "rewards/correctness_reward_func/std": 0.48795005679130554, "rewards/strict_format_reward_func/mean": 0.328125, "rewards/strict_format_reward_func/std": 0.23935678601264954, "step": 182, "time_profile/curr_logprobs_and_update": 0.2617022335034562, "time_profile/old_logps": 33.64694704860449, "time_profile/ref_logps": 33.735723273828626, "time_profile/reward": 0.0047216713428497314, "time_profile/text_rollout": 472.53653713781387, "train/gen_step": 11528.0, "und/advantages_abs_mean": 0.240478515625, "und/advantages_max": 1.8125, "und/advantages_mean": 0.0, "und/advantages_min": -0.6875, "und/advantages_std": 0.4047631025314331, "und/clip_ratio/high_mean": 0.00048385561603936367, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.00048385561603936367, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.69140625, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.21875, "und/kl": 0.0520571748202201, "und/loss": 0.002413793592950242, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 104.0, "und/prompt/min_length": 46.0, "und/reward": 0.4287109375, "und/reward_std": 0.46533018350601196 }, { "epoch": 2.3202562049639712, "grad_norm": 0.02421620301902294, "learning_rate": 1e-05, "loss": 0.0025, "step": 183, "time_profile/curr_logprobs_and_update": 0.2613777341321111, "train/gen_step": 11592.0, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0, "und/kl": 0.0602908979926724, "und/loss": 0.002848730291134416 }, { "epoch": 2.3330664531625303, "grad_norm": 0.024613449349999428, "learning_rate": 1e-05, "loss": 0.0025, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.2734375, "rewards/strict_format_reward_func/std": 0.250866562128067, "step": 184, "time_profile/curr_logprobs_and_update": 0.261491294571897, "time_profile/old_logps": 33.67325418256223, "time_profile/ref_logps": 33.7762045590207, "time_profile/reward": 0.004586059600114822, "time_profile/text_rollout": 473.8289050068706, "train/gen_step": 11656.0, "und/advantages_abs_mean": 0.174560546875, "und/advantages_max": 2.0625, "und/advantages_mean": 0.0, "und/advantages_min": -0.625, "und/advantages_std": 0.25663885474205017, "und/clip_ratio/high_mean": 0.0, "und/clip_ratio/low_mean": 5.7444853155175224e-05, "und/clip_ratio/region_mean": 5.7444853155175224e-05, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.603515625, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.15625, "und/kl": 0.06306225908338092, "und/loss": 0.00225927818718219, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 109.0, "und/prompt/min_length": 56.0, "und/reward": 0.3388671875, "und/reward_std": 0.2899373769760132 }, { "epoch": 2.345876701361089, "grad_norm": 0.0077704801224172115, "learning_rate": 1e-05, "loss": 0.0027, "step": 185, "time_profile/curr_logprobs_and_update": 0.2616247042315081, "train/gen_step": 11720.0, "und/clip_ratio/high_mean": 0.00011574074233067222, "und/clip_ratio/low_mean": 8.877841173671186e-05, "und/clip_ratio/region_mean": 0.00020451915406738408, "und/kl": 0.06688823399599642, "und/loss": 0.003165125963278115 }, { "epoch": 2.358686949559648, "grad_norm": 0.049835316836833954, "learning_rate": 1e-05, "loss": 0.0035, "rewards/correctness_reward_func/mean": 1.0625, "rewards/correctness_reward_func/std": 1.0059348344802856, "rewards/strict_format_reward_func/mean": 0.3984375, "rewards/strict_format_reward_func/std": 0.20275264978408813, "step": 186, "time_profile/curr_logprobs_and_update": 0.26167551841354, "time_profile/old_logps": 33.6585817206651, "time_profile/ref_logps": 33.735864114016294, "time_profile/reward": 0.00426170788705349, "time_profile/text_rollout": 470.94582207594067, "train/gen_step": 11784.0, "und/advantages_abs_mean": 0.705078125, "und/advantages_max": 2.0, "und/advantages_mean": 0.0, "und/advantages_min": -2.1875, "und/advantages_std": 0.8917018175125122, "und/clip_ratio/high_mean": 0.00022136736151878722, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.00022136736151878722, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.521484375, "und/completion/min_length": 220.0, "und/frac_reward_zero_std": 0.09375, "und/kl": 0.08871068802545778, "und/loss": 0.0025096102617681026, "und/prompt/max_length": 156.0, "und/prompt/mean_length": 99.0, "und/prompt/min_length": 42.0, "und/reward": 1.44921875, "und/reward_std": 1.124082326889038 }, { "epoch": 2.3714971977582064, "grad_norm": 0.07230977714061737, "learning_rate": 1e-05, "loss": 0.0038, "step": 187, "time_profile/curr_logprobs_and_update": 0.2613866949104704, "train/gen_step": 11848.0, "und/clip_ratio/high_mean": 0.00014307776291389018, "und/clip_ratio/low_mean": 6.300403038039804e-05, "und/clip_ratio/region_mean": 0.00020608179329428822, "und/kl": 0.097599101572996, "und/loss": 0.0026581829088172526 }, { "epoch": 2.3843074459567655, "grad_norm": 0.09674729406833649, "learning_rate": 1e-05, "loss": 0.0043, "rewards/correctness_reward_func/mean": 0.0, "rewards/correctness_reward_func/std": 0.0, "rewards/strict_format_reward_func/mean": 0.1796875, "rewards/strict_format_reward_func/std": 0.24180518090724945, "step": 188, "time_profile/curr_logprobs_and_update": 0.26152521098265424, "time_profile/old_logps": 33.65347420424223, "time_profile/ref_logps": 33.748919220641255, "time_profile/reward": 0.004438244737684727, "time_profile/text_rollout": 473.7467489670962, "train/gen_step": 11912.0, "und/advantages_abs_mean": 0.2021484375, "und/advantages_max": 0.4375, "und/advantages_mean": 0.0, "und/advantages_min": -0.375, "und/advantages_std": 0.22502446174621582, "und/clip_ratio/high_mean": 0.00026816331228474155, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.00026816331228474155, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.77734375, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.046875, "und/kl": 0.10714392748195678, "und/loss": 0.0022139963675726904, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 109.5, "und/prompt/min_length": 57.0, "und/reward": 0.1875, "und/reward_std": 0.2422981858253479 }, { "epoch": 2.397117694155324, "grad_norm": 0.013547060079872608, "learning_rate": 1e-05, "loss": 0.0071, "step": 189, "time_profile/curr_logprobs_and_update": 0.2615761066117557, "train/gen_step": 11976.0, "und/clip_ratio/high_mean": 5.278716344037093e-05, "und/clip_ratio/low_mean": 5.7444853155175224e-05, "und/clip_ratio/region_mean": 0.00011023201659554616, "und/kl": 0.17757004161830992, "und/loss": 0.0015241362062852204 }, { "epoch": 2.409927942353883, "grad_norm": 0.031346339732408524, "learning_rate": 1e-05, "loss": 0.0055, "rewards/correctness_reward_func/mean": 0.28125, "rewards/correctness_reward_func/std": 0.7007648944854736, "rewards/strict_format_reward_func/mean": 0.265625, "rewards/strict_format_reward_func/std": 0.2514837086200714, "step": 190, "time_profile/curr_logprobs_and_update": 0.2616291437589098, "time_profile/old_logps": 33.62939000222832, "time_profile/ref_logps": 33.71898835618049, "time_profile/reward": 0.0045094965025782585, "time_profile/text_rollout": 454.1341454349458, "train/gen_step": 12040.0, "und/advantages_abs_mean": 0.437255859375, "und/advantages_max": 2.1875, "und/advantages_mean": 0.0, "und/advantages_min": -1.5, "und/advantages_std": 0.6407745480537415, "und/clip_ratio/high_mean": 9.414017040398903e-05, "und/clip_ratio/low_mean": 0.00015810180047992617, "und/clip_ratio/region_mean": 0.0002522419781598728, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.822265625, "und/completion/min_length": 254.0, "und/frac_reward_zero_std": 0.140625, "und/kl": 0.1362051923351828, "und/loss": 0.002659691222561378, "und/prompt/max_length": 49.0, "und/prompt/mean_length": 46.5, "und/prompt/min_length": 44.0, "und/reward": 0.59375, "und/reward_std": 0.846889078617096 }, { "epoch": 2.422738190552442, "grad_norm": 0.026408294215798378, "learning_rate": 1e-05, "loss": 0.0049, "step": 191, "time_profile/curr_logprobs_and_update": 0.2613273377501173, "train/gen_step": 12104.0, "und/clip_ratio/high_mean": 0.00013889491310692392, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.00013889491310692392, "und/kl": 0.12136126437690109, "und/loss": 0.001747753280969988 }, { "epoch": 2.4355484387510007, "grad_norm": 0.03671589121222496, "learning_rate": 1e-05, "loss": 0.0026, "rewards/correctness_reward_func/mean": 0.40625, "rewards/correctness_reward_func/std": 0.8110105991363525, "rewards/strict_format_reward_func/mean": 0.34375, "rewards/strict_format_reward_func/std": 0.233588308095932, "step": 192, "time_profile/curr_logprobs_and_update": 0.2613590597611619, "time_profile/old_logps": 33.68849613983184, "time_profile/ref_logps": 33.76824611146003, "time_profile/reward": 0.004611174575984478, "time_profile/text_rollout": 472.31632216088474, "train/gen_step": 12168.0, "und/advantages_abs_mean": 0.55419921875, "und/advantages_max": 1.875, "und/advantages_mean": 0.0, "und/advantages_min": -1.75, "und/advantages_std": 0.6994109153747559, "und/clip_ratio/high_mean": 7.512019510613754e-05, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 7.512019510613754e-05, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.6640625, "und/completion/min_length": 210.0, "und/frac_reward_zero_std": 0.015625, "und/kl": 0.0662406939372886, "und/loss": 0.0016963802219152058, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 103.0, "und/prompt/min_length": 44.0, "und/reward": 0.8232421875, "und/reward_std": 0.9628657698631287 }, { "epoch": 2.4483586869495597, "grad_norm": 0.04320576414465904, "learning_rate": 1e-05, "loss": 0.0029, "step": 193, "time_profile/curr_logprobs_and_update": 0.26153383872588165, "train/gen_step": 12232.0, "und/clip_ratio/high_mean": 5.278716344037093e-05, "und/clip_ratio/low_mean": 4.882812572759576e-05, "und/clip_ratio/region_mean": 0.0001016152891679667, "und/kl": 0.07347345922607929, "und/loss": 0.0033696473110467196 }, { "epoch": 2.4611689351481187, "grad_norm": 0.023478908464312553, "learning_rate": 1e-05, "loss": 0.0034, "rewards/correctness_reward_func/mean": 0.28125, "rewards/correctness_reward_func/std": 0.7007648944854736, "rewards/strict_format_reward_func/mean": 0.3515625, "rewards/strict_format_reward_func/std": 0.2302463799715042, "step": 194, "time_profile/curr_logprobs_and_update": 0.2616785841237288, "time_profile/old_logps": 33.640625360421836, "time_profile/ref_logps": 33.778758310712874, "time_profile/reward": 0.004204129800200462, "time_profile/text_rollout": 472.5064763696864, "train/gen_step": 12296.0, "und/advantages_abs_mean": 0.38671875, "und/advantages_max": 1.8125, "und/advantages_mean": 0.0, "und/advantages_min": -0.9375, "und/advantages_std": 0.5569894909858704, "und/clip_ratio/high_mean": 0.0001396356092300266, "und/clip_ratio/low_mean": 0.00016529694767086767, "und/clip_ratio/region_mean": 0.00030493255690089427, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.46484375, "und/completion/min_length": 221.0, "und/frac_reward_zero_std": 0.078125, "und/kl": 0.08414672242361121, "und/loss": 0.0038269779179245234, "und/prompt/max_length": 156.0, "und/prompt/mean_length": 107.0, "und/prompt/min_length": 58.0, "und/reward": 0.5478515625, "und/reward_std": 0.6658832430839539 }, { "epoch": 2.4739791833466773, "grad_norm": 0.02270297333598137, "learning_rate": 1e-05, "loss": 0.0025, "step": 195, "time_profile/curr_logprobs_and_update": 0.26131977794284467, "train/gen_step": 12360.0, "und/clip_ratio/high_mean": 0.00011789133350248449, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.00011789133350248449, "und/kl": 0.06218441354576498, "und/loss": 0.0017657310236245394 }, { "epoch": 2.4867894315452364, "grad_norm": 0.04379241541028023, "learning_rate": 1e-05, "loss": 0.0024, "rewards/correctness_reward_func/mean": 0.5625, "rewards/correctness_reward_func/std": 0.9063270092010498, "rewards/strict_format_reward_func/mean": 0.328125, "rewards/strict_format_reward_func/std": 0.23935678601264954, "step": 196, "time_profile/curr_logprobs_and_update": 0.2615141822316218, "time_profile/old_logps": 33.65821726806462, "time_profile/ref_logps": 33.77625437732786, "time_profile/reward": 0.0044699013233184814, "time_profile/text_rollout": 489.89552684500813, "train/gen_step": 12424.0, "und/advantages_abs_mean": 0.632080078125, "und/advantages_max": 2.125, "und/advantages_mean": 0.0, "und/advantages_min": -1.875, "und/advantages_std": 0.8004096150398254, "und/clip_ratio/high_mean": 4.155584974796511e-05, "und/clip_ratio/low_mean": 5.5803571740398183e-05, "und/clip_ratio/region_mean": 9.73594214883633e-05, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.716796875, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.0, "und/kl": 0.05999593347951304, "und/loss": 0.0023087674635462463, "und/prompt/max_length": 162.0, "und/prompt/mean_length": 159.0, "und/prompt/min_length": 156.0, "und/reward": 0.759765625, "und/reward_std": 0.9920932054519653 }, { "epoch": 2.499599679743795, "grad_norm": 0.053688473999500275, "learning_rate": 1e-05, "loss": 0.0033, "step": 197, "time_profile/curr_logprobs_and_update": 0.26162081109941937, "train/gen_step": 12488.0, "und/clip_ratio/high_mean": 0.00011160714348079637, "und/clip_ratio/low_mean": 4.650297705666162e-05, "und/clip_ratio/region_mean": 0.000158110120537458, "und/kl": 0.08334491711866576, "und/loss": 0.002988875610753894 }, { "epoch": 2.512409927942354, "grad_norm": 0.010901146568357944, "learning_rate": 1e-05, "loss": 0.0144, "rewards/correctness_reward_func/mean": 0.84375, "rewards/correctness_reward_func/std": 0.9955257177352905, "rewards/strict_format_reward_func/mean": 0.3515625, "rewards/strict_format_reward_func/std": 0.2302463799715042, "step": 198, "time_profile/curr_logprobs_and_update": 0.2617834777047392, "time_profile/old_logps": 33.705376639030874, "time_profile/ref_logps": 33.77917596697807, "time_profile/reward": 0.004560342989861965, "time_profile/text_rollout": 456.2871036250144, "train/gen_step": 12552.0, "und/advantages_abs_mean": 0.375, "und/advantages_max": 1.625, "und/advantages_mean": 0.0, "und/advantages_min": -1.9375, "und/advantages_std": 0.5578123331069946, "und/clip_ratio/high_mean": 0.0004084461470483802, "und/clip_ratio/low_mean": 3.9062499126885086e-05, "und/clip_ratio/region_mean": 0.0004475086498132441, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.69921875, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.140625, "und/kl": 0.35732343784184195, "und/loss": 0.002203851705417037, "und/prompt/max_length": 62.0, "und/prompt/mean_length": 54.5, "und/prompt/min_length": 47.0, "und/reward": 1.15234375, "und/reward_std": 1.1305378675460815 }, { "epoch": 2.5252201761409125, "grad_norm": 0.03094113990664482, "learning_rate": 1e-05, "loss": 0.0023, "step": 199, "time_profile/curr_logprobs_and_update": 0.2613095892302226, "train/gen_step": 12616.0, "und/clip_ratio/high_mean": 0.0001487574481870979, "und/clip_ratio/low_mean": 0.0, "und/clip_ratio/region_mean": 0.0001487574481870979, "und/kl": 0.05986142602341715, "und/loss": 0.002186899437219836 }, { "epoch": 2.5380304243394716, "grad_norm": 0.03262926638126373, "learning_rate": 1e-05, "loss": 0.0021, "rewards/correctness_reward_func/mean": 0.0625, "rewards/correctness_reward_func/std": 0.35073620080947876, "rewards/strict_format_reward_func/mean": 0.0234375, "rewards/strict_format_reward_func/std": 0.10652101784944534, "step": 200, "time_profile/curr_logprobs_and_update": 0.26152220301446505, "time_profile/old_logps": 33.64662089664489, "time_profile/ref_logps": 33.763478820212185, "time_profile/reward": 0.004406346939504147, "time_profile/text_rollout": 519.7042249757797, "train/gen_step": 12680.0, "und/advantages_abs_mean": 0.123779296875, "und/advantages_max": 2.1875, "und/advantages_mean": 0.0, "und/advantages_min": -0.5625, "und/advantages_std": 0.36074504256248474, "und/clip_ratio/high_mean": 5.139802669873461e-05, "und/clip_ratio/low_mean": 5.139802669873461e-05, "und/clip_ratio/region_mean": 0.00010279605339746922, "und/completion/max_length": 255.0, "und/completion/mean_length": 254.958984375, "und/completion/min_length": 253.0, "und/frac_reward_zero_std": 0.640625, "und/kl": 0.05215715864324011, "und/loss": 0.0015407809119096783, "und/prompt/max_length": 247.0, "und/prompt/mean_length": 247.0, "und/prompt/min_length": 247.0, "und/reward": 0.072265625, "und/reward_std": 0.3826628625392914 } ], "logging_steps": 1.0, "max_steps": 790, "num_input_tokens_seen": 0, "num_train_epochs": 10, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }