| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 2.5380304243394716, |
| "eval_steps": 500, |
| "global_step": 200, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.012810248198558846, |
| "grad_norm": 0.011678964830935001, |
| "learning_rate": 0.0, |
| "loss": 0.0, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.234375, |
| "rewards/strict_format_reward_func/std": 0.2514837086200714, |
| "step": 1, |
| "time_profile/curr_logprobs_and_update": 0.2617394689877983, |
| "time_profile/old_logps": 33.7158753760159, |
| "time_profile/ref_logps": 33.76804309990257, |
| "time_profile/reward": 0.0047389911487698555, |
| "time_profile/text_rollout": 474.3757369443774, |
| "train/gen_step": 64.0, |
| "und/advantages_abs_mean": 0.2001953125, |
| "und/advantages_max": 2.125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.4375, |
| "und/advantages_std": 0.2579461336135864, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.751953125, |
| "und/completion/min_length": 250.0, |
| "und/frac_reward_zero_std": 0.046875, |
| "und/kl": 0.0, |
| "und/loss": 0.0, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 107.0, |
| "und/prompt/min_length": 52.0, |
| "und/reward": 0.193359375, |
| "und/reward_std": 0.28102907538414 |
| }, |
| { |
| "epoch": 0.025620496397117692, |
| "grad_norm": 0.00965250376611948, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "step": 2, |
| "time_profile/curr_logprobs_and_update": 0.26145937727415003, |
| "train/gen_step": 128.0, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/kl": 0.0, |
| "und/loss": 0.0 |
| }, |
| { |
| "epoch": 0.03843074459567654, |
| "grad_norm": 0.04503251612186432, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "rewards/correctness_reward_func/mean": 0.4375, |
| "rewards/correctness_reward_func/std": 0.8333333730697632, |
| "rewards/strict_format_reward_func/mean": 0.15625, |
| "rewards/strict_format_reward_func/std": 0.233588308095932, |
| "step": 3, |
| "time_profile/curr_logprobs_and_update": 0.2616513767570723, |
| "time_profile/old_logps": 33.65838770195842, |
| "time_profile/ref_logps": 33.75480777770281, |
| "time_profile/reward": 0.004094251431524754, |
| "time_profile/text_rollout": 504.14777049794793, |
| "train/gen_step": 192.0, |
| "und/advantages_abs_mean": 0.443603515625, |
| "und/advantages_max": 2.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.6875, |
| "und/advantages_std": 0.7080574631690979, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.79296875, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.421875, |
| "und/kl": 0.00019532828423507453, |
| "und/loss": 7.946068224740088e-06, |
| "und/prompt/max_length": 247.0, |
| "und/prompt/mean_length": 201.5, |
| "und/prompt/min_length": 156.0, |
| "und/reward": 0.5078125, |
| "und/reward_std": 0.9087191820144653 |
| }, |
| { |
| "epoch": 0.051240992794235385, |
| "grad_norm": 0.04360778629779816, |
| "learning_rate": 1e-05, |
| "loss": 0.0001, |
| "step": 4, |
| "time_profile/curr_logprobs_and_update": 0.261664173391182, |
| "train/gen_step": 256.0, |
| "und/clip_ratio/high_mean": 0.0003149772164761089, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0003149772164761089, |
| "und/kl": 0.00025711835689889995, |
| "und/loss": 0.000590196544485444 |
| }, |
| { |
| "epoch": 0.06405124099279423, |
| "grad_norm": 0.005336050409823656, |
| "learning_rate": 1e-05, |
| "loss": 0.0001, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.0625, |
| "rewards/strict_format_reward_func/std": 0.1666666716337204, |
| "step": 5, |
| "time_profile/curr_logprobs_and_update": 0.2615119785768911, |
| "time_profile/old_logps": 33.64620256703347, |
| "time_profile/ref_logps": 33.72650748398155, |
| "time_profile/reward": 0.004272850230336189, |
| "time_profile/text_rollout": 491.0981372119859, |
| "train/gen_step": 320.0, |
| "und/advantages_abs_mean": 0.095703125, |
| "und/advantages_max": 2.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.375, |
| "und/advantages_std": 0.21501831710338593, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.9296875, |
| "und/completion/min_length": 252.0, |
| "und/frac_reward_zero_std": 0.5, |
| "und/kl": 0.00318289036204078, |
| "und/loss": 0.00012254955510471177, |
| "und/prompt/max_length": 247.0, |
| "und/prompt/mean_length": 152.5, |
| "und/prompt/min_length": 58.0, |
| "und/reward": 0.06640625, |
| "und/reward_std": 0.23300214111804962 |
| }, |
| { |
| "epoch": 0.07686148919135308, |
| "grad_norm": 0.006930716801434755, |
| "learning_rate": 1e-05, |
| "loss": 0.0003, |
| "step": 6, |
| "time_profile/curr_logprobs_and_update": 0.26136891632631887, |
| "train/gen_step": 384.0, |
| "und/clip_ratio/high_mean": 0.0004957187084073666, |
| "und/clip_ratio/low_mean": 0.00018175119475927204, |
| "und/clip_ratio/region_mean": 0.0006774698958906811, |
| "und/kl": 0.0036970916280552046, |
| "und/loss": 5.486323918546532e-05 |
| }, |
| { |
| "epoch": 0.08967173738991192, |
| "grad_norm": 0.012152941897511482, |
| "learning_rate": 1e-05, |
| "loss": 0.0001, |
| "rewards/correctness_reward_func/mean": 0.03125, |
| "rewards/correctness_reward_func/std": 0.25, |
| "rewards/strict_format_reward_func/mean": 0.15625, |
| "rewards/strict_format_reward_func/std": 0.233588308095932, |
| "step": 7, |
| "time_profile/curr_logprobs_and_update": 0.26169058527739253, |
| "time_profile/old_logps": 33.673401176929474, |
| "time_profile/ref_logps": 33.76787843555212, |
| "time_profile/reward": 0.00381280854344368, |
| "time_profile/text_rollout": 473.39493833016604, |
| "train/gen_step": 448.0, |
| "und/advantages_abs_mean": 0.197509765625, |
| "und/advantages_max": 2.0625, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.5625, |
| "und/advantages_std": 0.2547253966331482, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.755859375, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.0625, |
| "und/kl": 0.0018817147374647902, |
| "und/loss": 4.349416121840477e-05, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 107.0, |
| "und/prompt/min_length": 52.0, |
| "und/reward": 0.1689453125, |
| "und/reward_std": 0.27497100830078125 |
| }, |
| { |
| "epoch": 0.10248198558847077, |
| "grad_norm": 0.011709543876349926, |
| "learning_rate": 1e-05, |
| "loss": 0.0001, |
| "step": 8, |
| "time_profile/curr_logprobs_and_update": 0.2615692205145024, |
| "train/gen_step": 512.0, |
| "und/clip_ratio/high_mean": 0.00020409237549756654, |
| "und/clip_ratio/low_mean": 0.00016332855375367217, |
| "und/clip_ratio/region_mean": 0.0003674209328892175, |
| "und/kl": 0.0024647556260788406, |
| "und/loss": -4.9581576604396105e-05 |
| }, |
| { |
| "epoch": 0.11529223378702963, |
| "grad_norm": 0.05086541920900345, |
| "learning_rate": 1e-05, |
| "loss": 0.0001, |
| "rewards/correctness_reward_func/mean": 0.6875, |
| "rewards/correctness_reward_func/std": 0.9574271440505981, |
| "rewards/strict_format_reward_func/mean": 0.2734375, |
| "rewards/strict_format_reward_func/std": 0.250866562128067, |
| "step": 9, |
| "time_profile/curr_logprobs_and_update": 0.2616978839650983, |
| "time_profile/old_logps": 33.66389245353639, |
| "time_profile/ref_logps": 33.785877684131265, |
| "time_profile/reward": 0.004252792336046696, |
| "time_profile/text_rollout": 488.8415644085035, |
| "train/gen_step": 576.0, |
| "und/advantages_abs_mean": 0.7744140625, |
| "und/advantages_max": 2.0625, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -2.0625, |
| "und/advantages_std": 0.9534159302711487, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.6171875, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.0, |
| "und/kl": 0.0022935827773835626, |
| "und/loss": 0.00010531611042097211, |
| "und/prompt/max_length": 156.0, |
| "und/prompt/mean_length": 156.0, |
| "und/prompt/min_length": 156.0, |
| "und/reward": 1.041015625, |
| "und/reward_std": 1.1144300699234009 |
| }, |
| { |
| "epoch": 0.12810248198558846, |
| "grad_norm": 0.07067691534757614, |
| "learning_rate": 1e-05, |
| "loss": 0.0002, |
| "step": 10, |
| "time_profile/curr_logprobs_and_update": 0.26148990585352294, |
| "train/gen_step": 640.0, |
| "und/clip_ratio/high_mean": 0.000207764285732992, |
| "und/clip_ratio/low_mean": 0.00026153815633733757, |
| "und/clip_ratio/region_mean": 0.00046930244570830837, |
| "und/kl": 0.0025965598733819206, |
| "und/loss": -0.00032083288533613086 |
| }, |
| { |
| "epoch": 0.14091273018414732, |
| "grad_norm": 0.011798444204032421, |
| "learning_rate": 1e-05, |
| "loss": 0.0001, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.3359375, |
| "rewards/strict_format_reward_func/std": 0.2366211861371994, |
| "step": 11, |
| "time_profile/curr_logprobs_and_update": 0.2613880275603151, |
| "time_profile/old_logps": 33.668298315256834, |
| "time_profile/ref_logps": 33.73314023669809, |
| "time_profile/reward": 0.0044129379093647, |
| "time_profile/text_rollout": 454.3680015280843, |
| "train/gen_step": 704.0, |
| "und/advantages_abs_mean": 0.1396484375, |
| "und/advantages_max": 1.75, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.4375, |
| "und/advantages_std": 0.20029333233833313, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.6640625, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.28125, |
| "und/kl": 0.0030043681945244316, |
| "und/loss": 4.8866542471159846e-05, |
| "und/prompt/max_length": 49.0, |
| "und/prompt/mean_length": 49.0, |
| "und/prompt/min_length": 49.0, |
| "und/reward": 0.34765625, |
| "und/reward_std": 0.25070229172706604 |
| }, |
| { |
| "epoch": 0.15372297838270615, |
| "grad_norm": 0.00986644346266985, |
| "learning_rate": 1e-05, |
| "loss": 0.0002, |
| "step": 12, |
| "time_profile/curr_logprobs_and_update": 0.2613714834296843, |
| "train/gen_step": 768.0, |
| "und/clip_ratio/high_mean": 0.0003061741153942421, |
| "und/clip_ratio/low_mean": 0.0004524241703620646, |
| "und/clip_ratio/region_mean": 0.0007585982857563067, |
| "und/kl": 0.0042751526357278635, |
| "und/loss": 0.0005581114168933254 |
| }, |
| { |
| "epoch": 0.16653322658126501, |
| "grad_norm": 0.044079430401325226, |
| "learning_rate": 1e-05, |
| "loss": 0.0002, |
| "rewards/correctness_reward_func/mean": 0.78125, |
| "rewards/correctness_reward_func/std": 0.983494758605957, |
| "rewards/strict_format_reward_func/mean": 0.375, |
| "rewards/strict_format_reward_func/std": 0.2182178944349289, |
| "step": 13, |
| "time_profile/curr_logprobs_and_update": 0.2613269358407706, |
| "time_profile/old_logps": 33.61356293410063, |
| "time_profile/ref_logps": 33.69876565504819, |
| "time_profile/reward": 0.004282657988369465, |
| "time_profile/text_rollout": 455.1681863386184, |
| "train/gen_step": 832.0, |
| "und/advantages_abs_mean": 0.33984375, |
| "und/advantages_max": 1.0625, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -2.1875, |
| "und/advantages_std": 0.5323442816734314, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.69140625, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.171875, |
| "und/kl": 0.00535378480435611, |
| "und/loss": 0.00024319114163517952, |
| "und/prompt/max_length": 55.0, |
| "und/prompt/mean_length": 51.0, |
| "und/prompt/min_length": 47.0, |
| "und/reward": 1.21875, |
| "und/reward_std": 1.1256657838821411 |
| }, |
| { |
| "epoch": 0.17934347477982385, |
| "grad_norm": 0.04768079146742821, |
| "learning_rate": 1e-05, |
| "loss": 0.0001, |
| "step": 14, |
| "time_profile/curr_logprobs_and_update": 0.2612948026799131, |
| "train/gen_step": 896.0, |
| "und/clip_ratio/high_mean": 0.0003154302030452527, |
| "und/clip_ratio/low_mean": 0.0004365383902040776, |
| "und/clip_ratio/region_mean": 0.0007519685932493303, |
| "und/kl": 0.005003000936994795, |
| "und/loss": -6.164377555251122e-05 |
| }, |
| { |
| "epoch": 0.1921537229783827, |
| "grad_norm": 0.013837647624313831, |
| "learning_rate": 1e-05, |
| "loss": 0.0001, |
| "rewards/correctness_reward_func/mean": 0.03125, |
| "rewards/correctness_reward_func/std": 0.25, |
| "rewards/strict_format_reward_func/mean": 0.3203125, |
| "rewards/strict_format_reward_func/std": 0.24180518090724945, |
| "step": 15, |
| "time_profile/curr_logprobs_and_update": 0.261544900276931, |
| "time_profile/old_logps": 33.64624940324575, |
| "time_profile/ref_logps": 33.727370373904705, |
| "time_profile/reward": 0.004188773222267628, |
| "time_profile/text_rollout": 454.6840885374695, |
| "train/gen_step": 960.0, |
| "und/advantages_abs_mean": 0.149658203125, |
| "und/advantages_max": 1.9375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.5625, |
| "und/advantages_std": 0.2218771129846573, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.619140625, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.203125, |
| "und/kl": 0.002458287915942492, |
| "und/loss": 0.00015533823614077846, |
| "und/prompt/max_length": 51.0, |
| "und/prompt/mean_length": 49.5, |
| "und/prompt/min_length": 48.0, |
| "und/reward": 0.3330078125, |
| "und/reward_std": 0.2671593129634857 |
| }, |
| { |
| "epoch": 0.20496397117694154, |
| "grad_norm": 0.012836061418056488, |
| "learning_rate": 1e-05, |
| "loss": 0.0001, |
| "step": 16, |
| "time_profile/curr_logprobs_and_update": 0.2614677896053763, |
| "train/gen_step": 1024.0, |
| "und/clip_ratio/high_mean": 0.0010820379393408075, |
| "und/clip_ratio/low_mean": 5.425347262644209e-05, |
| "und/clip_ratio/region_mean": 0.0011362914119672496, |
| "und/kl": 0.0026947305022986257, |
| "und/loss": 1.8947941015312608e-05 |
| }, |
| { |
| "epoch": 0.2177742193755004, |
| "grad_norm": 0.03666432574391365, |
| "learning_rate": 1e-05, |
| "loss": 0.0002, |
| "rewards/correctness_reward_func/mean": 0.125, |
| "rewards/correctness_reward_func/std": 0.48795005679130554, |
| "rewards/strict_format_reward_func/mean": 0.484375, |
| "rewards/strict_format_reward_func/std": 0.08768405020236969, |
| "step": 17, |
| "time_profile/curr_logprobs_and_update": 0.26129884374677204, |
| "time_profile/old_logps": 33.58372610062361, |
| "time_profile/ref_logps": 33.71312084700912, |
| "time_profile/reward": 0.004377733916044235, |
| "time_profile/text_rollout": 453.4618791854009, |
| "train/gen_step": 1088.0, |
| "und/advantages_abs_mean": 0.24462890625, |
| "und/advantages_max": 1.875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.9375, |
| "und/advantages_std": 0.4919235110282898, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.58984375, |
| "und/completion/min_length": 216.0, |
| "und/frac_reward_zero_std": 0.453125, |
| "und/kl": 0.003914871423148725, |
| "und/loss": 0.00013992213200708647, |
| "und/prompt/max_length": 45.0, |
| "und/prompt/mean_length": 44.0, |
| "und/prompt/min_length": 43.0, |
| "und/reward": 0.6201171875, |
| "und/reward_std": 0.5268946886062622 |
| }, |
| { |
| "epoch": 0.23058446757405926, |
| "grad_norm": 0.04468945786356926, |
| "learning_rate": 1e-05, |
| "loss": 0.0003, |
| "step": 18, |
| "time_profile/curr_logprobs_and_update": 0.26125960882927757, |
| "train/gen_step": 1152.0, |
| "und/clip_ratio/high_mean": 0.00032832663418957964, |
| "und/clip_ratio/low_mean": 0.0001570866115798708, |
| "und/clip_ratio/region_mean": 0.00048541324576945044, |
| "und/kl": 0.004693901622886187, |
| "und/loss": 0.0001238020138245588 |
| }, |
| { |
| "epoch": 0.2433947157726181, |
| "grad_norm": 0.04278712719678879, |
| "learning_rate": 1e-05, |
| "loss": 0.0002, |
| "rewards/correctness_reward_func/mean": 0.34375, |
| "rewards/correctness_reward_func/std": 0.7605084180831909, |
| "rewards/strict_format_reward_func/mean": 0.4140625, |
| "rewards/strict_format_reward_func/std": 0.19012710452079773, |
| "step": 19, |
| "time_profile/curr_logprobs_and_update": 0.2615380200877553, |
| "time_profile/old_logps": 33.66465362161398, |
| "time_profile/ref_logps": 33.743118970654905, |
| "time_profile/reward": 0.00426104012876749, |
| "time_profile/text_rollout": 454.9674591496587, |
| "train/gen_step": 1216.0, |
| "und/advantages_abs_mean": 0.5419921875, |
| "und/advantages_max": 1.875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.6875, |
| "und/advantages_std": 0.6924689412117004, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.603515625, |
| "und/completion/min_length": 247.0, |
| "und/frac_reward_zero_std": 0.0, |
| "und/kl": 0.005255785068584373, |
| "und/loss": 0.000339222839102149, |
| "und/prompt/max_length": 53.0, |
| "und/prompt/mean_length": 52.0, |
| "und/prompt/min_length": 51.0, |
| "und/reward": 0.7529296875, |
| "und/reward_std": 0.8764092326164246 |
| }, |
| { |
| "epoch": 0.2562049639711769, |
| "grad_norm": 0.05156658962368965, |
| "learning_rate": 1e-05, |
| "loss": 0.0005, |
| "step": 20, |
| "time_profile/curr_logprobs_and_update": 0.2612772498978302, |
| "train/gen_step": 1280.0, |
| "und/clip_ratio/high_mean": 0.000682802070514299, |
| "und/clip_ratio/low_mean": 0.0004532934799499344, |
| "und/clip_ratio/region_mean": 0.0011360955541022122, |
| "und/kl": 0.007088688389558229, |
| "und/loss": -0.000385153922252357 |
| }, |
| { |
| "epoch": 0.2690152121697358, |
| "grad_norm": 0.010599356144666672, |
| "learning_rate": 1e-05, |
| "loss": 0.0004, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.03125, |
| "rewards/strict_format_reward_func/std": 0.12198751419782639, |
| "step": 21, |
| "time_profile/curr_logprobs_and_update": 0.26165466259408277, |
| "time_profile/old_logps": 33.66928117442876, |
| "time_profile/ref_logps": 33.74430500715971, |
| "time_profile/reward": 0.005226008594036102, |
| "time_profile/text_rollout": 505.46357171423733, |
| "train/gen_step": 1344.0, |
| "und/advantages_abs_mean": 0.0986328125, |
| "und/advantages_max": 2.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.375, |
| "und/advantages_std": 0.23039571940898895, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.90625, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.546875, |
| "und/kl": 0.009255951059458312, |
| "und/loss": 0.0003061078954686991, |
| "und/prompt/max_length": 247.0, |
| "und/prompt/mean_length": 204.5, |
| "und/prompt/min_length": 162.0, |
| "und/reward": 0.068359375, |
| "und/reward_std": 0.2506641745567322 |
| }, |
| { |
| "epoch": 0.28182546036829464, |
| "grad_norm": 0.006633961573243141, |
| "learning_rate": 1e-05, |
| "loss": 0.0005, |
| "step": 22, |
| "time_profile/curr_logprobs_and_update": 0.2616323969559744, |
| "train/gen_step": 1408.0, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.00017277644656132907, |
| "und/clip_ratio/region_mean": 0.00017277644656132907, |
| "und/kl": 0.01153348765183182, |
| "und/loss": 0.0003008711780587703 |
| }, |
| { |
| "epoch": 0.2946357085668535, |
| "grad_norm": 0.01536457147449255, |
| "learning_rate": 1e-05, |
| "loss": 0.0005, |
| "rewards/correctness_reward_func/mean": 0.90625, |
| "rewards/correctness_reward_func/std": 1.003466248512268, |
| "rewards/strict_format_reward_func/mean": 0.34375, |
| "rewards/strict_format_reward_func/std": 0.233588308095932, |
| "step": 23, |
| "time_profile/curr_logprobs_and_update": 0.26142213236016687, |
| "time_profile/old_logps": 33.64063487108797, |
| "time_profile/ref_logps": 33.71632799040526, |
| "time_profile/reward": 0.004507049918174744, |
| "time_profile/text_rollout": 456.5940729761496, |
| "train/gen_step": 1472.0, |
| "und/advantages_abs_mean": 0.3193359375, |
| "und/advantages_max": 1.0, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -2.1875, |
| "und/advantages_std": 0.508188784122467, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.421875, |
| "und/completion/min_length": 126.0, |
| "und/frac_reward_zero_std": 0.109375, |
| "und/kl": 0.011904407194379019, |
| "und/loss": 0.000405974641296325, |
| "und/prompt/max_length": 68.0, |
| "und/prompt/mean_length": 57.0, |
| "und/prompt/min_length": 46.0, |
| "und/reward": 1.1806640625, |
| "und/reward_std": 1.1607900857925415 |
| }, |
| { |
| "epoch": 0.3074459567654123, |
| "grad_norm": 0.017077524214982986, |
| "learning_rate": 1e-05, |
| "loss": 0.0007, |
| "step": 24, |
| "time_profile/curr_logprobs_and_update": 0.26139352870814037, |
| "train/gen_step": 1536.0, |
| "und/clip_ratio/high_mean": 0.0003636985675257165, |
| "und/clip_ratio/low_mean": 0.000808012075140141, |
| "und/clip_ratio/region_mean": 0.0011717106426658574, |
| "und/kl": 0.0162865712918574, |
| "und/loss": 0.0004572992978353341 |
| }, |
| { |
| "epoch": 0.32025620496397117, |
| "grad_norm": 0.011341015808284283, |
| "learning_rate": 1e-05, |
| "loss": 0.0009, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.15625, |
| "rewards/strict_format_reward_func/std": 0.233588308095932, |
| "step": 25, |
| "time_profile/curr_logprobs_and_update": 0.2614934611192439, |
| "time_profile/old_logps": 33.66488476470113, |
| "time_profile/ref_logps": 33.74662786722183, |
| "time_profile/reward": 0.0043584490194916725, |
| "time_profile/text_rollout": 473.53795502893627, |
| "train/gen_step": 1600.0, |
| "und/advantages_abs_mean": 0.18212890625, |
| "und/advantages_max": 2.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.3125, |
| "und/advantages_std": 0.2327723652124405, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.763671875, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.078125, |
| "und/kl": 0.02267802683309128, |
| "und/loss": 0.001361295189667544, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 109.5, |
| "und/prompt/min_length": 57.0, |
| "und/reward": 0.14453125, |
| "und/reward_std": 0.2475108504295349 |
| }, |
| { |
| "epoch": 0.33306645316253003, |
| "grad_norm": 0.014138264581561089, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "step": 26, |
| "time_profile/curr_logprobs_and_update": 0.2614813649561256, |
| "train/gen_step": 1664.0, |
| "und/clip_ratio/high_mean": 9.385726662003435e-05, |
| "und/clip_ratio/low_mean": 0.0006062783650122583, |
| "und/clip_ratio/region_mean": 0.0007001356316322926, |
| "und/kl": 0.03298538580929744, |
| "und/loss": 0.0003666753910565035 |
| }, |
| { |
| "epoch": 0.3458767013610889, |
| "grad_norm": 0.036602944135665894, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "rewards/correctness_reward_func/mean": 0.15625, |
| "rewards/correctness_reward_func/std": 0.5409794449806213, |
| "rewards/strict_format_reward_func/mean": 0.296875, |
| "rewards/strict_format_reward_func/std": 0.24750742316246033, |
| "step": 27, |
| "time_profile/curr_logprobs_and_update": 0.26141884991375264, |
| "time_profile/old_logps": 33.63414192106575, |
| "time_profile/ref_logps": 33.71741700544953, |
| "time_profile/reward": 0.004512332379817963, |
| "time_profile/text_rollout": 454.9066513767466, |
| "train/gen_step": 1728.0, |
| "und/advantages_abs_mean": 0.361572265625, |
| "und/advantages_max": 2.0, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.25, |
| "und/advantages_std": 0.5393062829971313, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.65234375, |
| "und/completion/min_length": 250.0, |
| "und/frac_reward_zero_std": 0.015625, |
| "und/kl": 0.04265276483056368, |
| "und/loss": 0.0006861621513962746, |
| "und/prompt/max_length": 59.0, |
| "und/prompt/mean_length": 52.0, |
| "und/prompt/min_length": 45.0, |
| "und/reward": 0.44921875, |
| "und/reward_std": 0.6556687951087952 |
| }, |
| { |
| "epoch": 0.3586869495596477, |
| "grad_norm": 0.05098080635070801, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "step": 28, |
| "time_profile/curr_logprobs_and_update": 0.261381899996195, |
| "train/gen_step": 1792.0, |
| "und/clip_ratio/high_mean": 0.0009093214357562829, |
| "und/clip_ratio/low_mean": 0.0001023065487970598, |
| "und/clip_ratio/region_mean": 0.0010116279845533427, |
| "und/kl": 0.037396987856482156, |
| "und/loss": 0.0021928895730525255 |
| }, |
| { |
| "epoch": 0.37149719775820655, |
| "grad_norm": 0.009760422632098198, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.2734375, |
| "rewards/strict_format_reward_func/std": 0.250866562128067, |
| "step": 29, |
| "time_profile/curr_logprobs_and_update": 0.26138593546056654, |
| "time_profile/old_logps": 33.63254508841783, |
| "time_profile/ref_logps": 33.774293441325426, |
| "time_profile/reward": 0.004322300665080547, |
| "time_profile/text_rollout": 455.36977090034634, |
| "train/gen_step": 1856.0, |
| "und/advantages_abs_mean": 0.2197265625, |
| "und/advantages_max": 0.4375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.4375, |
| "und/advantages_std": 0.23460420966148376, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.712890625, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.015625, |
| "und/kl": 0.056478848066035425, |
| "und/loss": 0.006321181455859914, |
| "und/prompt/max_length": 54.0, |
| "und/prompt/mean_length": 53.5, |
| "und/prompt/min_length": 53.0, |
| "und/reward": 0.236328125, |
| "und/reward_std": 0.24987001717090607 |
| }, |
| { |
| "epoch": 0.3843074459567654, |
| "grad_norm": 0.010228869505226612, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "step": 30, |
| "time_profile/curr_logprobs_and_update": 0.2613552274706308, |
| "train/gen_step": 1920.0, |
| "und/clip_ratio/high_mean": 0.0011887354085047264, |
| "und/clip_ratio/low_mean": 4.245923992129974e-05, |
| "und/clip_ratio/region_mean": 0.001231194648426026, |
| "und/kl": 0.023746514114463935, |
| "und/loss": 0.0005230403039604425 |
| }, |
| { |
| "epoch": 0.3971176941553243, |
| "grad_norm": 0.050902411341667175, |
| "learning_rate": 1e-05, |
| "loss": 0.0008, |
| "rewards/correctness_reward_func/mean": 0.5, |
| "rewards/correctness_reward_func/std": 0.8728715777397156, |
| "rewards/strict_format_reward_func/mean": 0.34375, |
| "rewards/strict_format_reward_func/std": 0.233588308095932, |
| "step": 31, |
| "time_profile/curr_logprobs_and_update": 0.26159091785666533, |
| "time_profile/old_logps": 33.64319808781147, |
| "time_profile/ref_logps": 33.71481241378933, |
| "time_profile/reward": 0.004479923285543919, |
| "time_profile/text_rollout": 473.2088502328843, |
| "train/gen_step": 1984.0, |
| "und/advantages_abs_mean": 0.535400390625, |
| "und/advantages_max": 1.5625, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.75, |
| "und/advantages_std": 0.6867490410804749, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.625, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.015625, |
| "und/kl": 0.018857156013837084, |
| "und/loss": 0.0008609343785792589, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 108.0, |
| "und/prompt/min_length": 54.0, |
| "und/reward": 0.83203125, |
| "und/reward_std": 1.0073648691177368 |
| }, |
| { |
| "epoch": 0.4099279423538831, |
| "grad_norm": 0.06242002546787262, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "step": 32, |
| "time_profile/curr_logprobs_and_update": 0.26177428937808145, |
| "train/gen_step": 2048.0, |
| "und/clip_ratio/high_mean": 0.0007951511470309924, |
| "und/clip_ratio/low_mean": 0.00020771568233612925, |
| "und/clip_ratio/region_mean": 0.0010028668257291429, |
| "und/kl": 0.03375891799441888, |
| "und/loss": 0.000749626662582159 |
| }, |
| { |
| "epoch": 0.42273819055244194, |
| "grad_norm": 0.04212071746587753, |
| "learning_rate": 1e-05, |
| "loss": 0.0006, |
| "rewards/correctness_reward_func/mean": 0.5, |
| "rewards/correctness_reward_func/std": 0.8728715777397156, |
| "rewards/strict_format_reward_func/mean": 0.3125, |
| "rewards/strict_format_reward_func/std": 0.24397502839565277, |
| "step": 33, |
| "time_profile/curr_logprobs_and_update": 0.26152711105532944, |
| "time_profile/old_logps": 33.65837723109871, |
| "time_profile/ref_logps": 33.75508733931929, |
| "time_profile/reward": 0.0049184514209628105, |
| "time_profile/text_rollout": 472.5054632341489, |
| "train/gen_step": 2112.0, |
| "und/advantages_abs_mean": 0.53515625, |
| "und/advantages_max": 1.75, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.75, |
| "und/advantages_std": 0.6916294097900391, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.671875, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.015625, |
| "und/kl": 0.015675050650315825, |
| "und/loss": 0.0005543280858546495, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 104.5, |
| "und/prompt/min_length": 47.0, |
| "und/reward": 0.84375, |
| "und/reward_std": 1.0150530338287354 |
| }, |
| { |
| "epoch": 0.4355484387510008, |
| "grad_norm": 0.0346144400537014, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "step": 34, |
| "time_profile/curr_logprobs_and_update": 0.2614700558333425, |
| "train/gen_step": 2176.0, |
| "und/clip_ratio/high_mean": 0.0002005133756028954, |
| "und/clip_ratio/low_mean": 0.00024482345179421827, |
| "und/clip_ratio/region_mean": 0.00044533682739711367, |
| "und/kl": 0.025506131180009106, |
| "und/loss": 0.001075362495612353 |
| }, |
| { |
| "epoch": 0.44835868694955966, |
| "grad_norm": 0.04864125698804855, |
| "learning_rate": 1e-05, |
| "loss": 0.0008, |
| "rewards/correctness_reward_func/mean": 0.1875, |
| "rewards/correctness_reward_func/std": 0.5875696539878845, |
| "rewards/strict_format_reward_func/mean": 0.25, |
| "rewards/strict_format_reward_func/std": 0.2519763112068176, |
| "step": 35, |
| "time_profile/curr_logprobs_and_update": 0.2617200283857528, |
| "time_profile/old_logps": 33.63409499172121, |
| "time_profile/ref_logps": 33.74282674957067, |
| "time_profile/reward": 0.004408497363328934, |
| "time_profile/text_rollout": 487.6969060283154, |
| "train/gen_step": 2240.0, |
| "und/advantages_abs_mean": 0.337890625, |
| "und/advantages_max": 1.875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.4375, |
| "und/advantages_std": 0.585414707660675, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.32421875, |
| "und/completion/min_length": 175.0, |
| "und/frac_reward_zero_std": 0.46875, |
| "und/kl": 0.019317339225381147, |
| "und/loss": 0.0008824675582559394, |
| "und/prompt/max_length": 247.0, |
| "und/prompt/mean_length": 145.0, |
| "und/prompt/min_length": 43.0, |
| "und/reward": 0.478515625, |
| "und/reward_std": 0.7729067802429199 |
| }, |
| { |
| "epoch": 0.4611689351481185, |
| "grad_norm": 0.043183691799640656, |
| "learning_rate": 1e-05, |
| "loss": 0.0009, |
| "step": 36, |
| "time_profile/curr_logprobs_and_update": 0.26155611700960435, |
| "train/gen_step": 2304.0, |
| "und/clip_ratio/high_mean": 0.00021402155107352883, |
| "und/clip_ratio/low_mean": 0.00021690024368581362, |
| "und/clip_ratio/region_mean": 0.00043092179839732125, |
| "und/kl": 0.024045594029303174, |
| "und/loss": 0.0011579781168506997 |
| }, |
| { |
| "epoch": 0.4739791833466773, |
| "grad_norm": 0.010638823732733727, |
| "learning_rate": 1e-05, |
| "loss": 0.0009, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.109375, |
| "rewards/strict_format_reward_func/std": 0.2083333432674408, |
| "step": 37, |
| "time_profile/curr_logprobs_and_update": 0.2616240307397675, |
| "time_profile/old_logps": 33.59795498009771, |
| "time_profile/ref_logps": 33.70516960695386, |
| "time_profile/reward": 0.004617785103619099, |
| "time_profile/text_rollout": 487.4186108149588, |
| "train/gen_step": 2368.0, |
| "und/advantages_abs_mean": 0.136962890625, |
| "und/advantages_max": 2.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.625, |
| "und/advantages_std": 0.25544461607933044, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.880859375, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.4375, |
| "und/kl": 0.022708220010827063, |
| "und/loss": 0.0009557972493325906, |
| "und/prompt/max_length": 247.0, |
| "und/prompt/mean_length": 151.0, |
| "und/prompt/min_length": 55.0, |
| "und/reward": 0.1318359375, |
| "und/reward_std": 0.2962620258331299 |
| }, |
| { |
| "epoch": 0.4867894315452362, |
| "grad_norm": 0.010083939880132675, |
| "learning_rate": 1e-05, |
| "loss": 0.001, |
| "step": 38, |
| "time_profile/curr_logprobs_and_update": 0.2614540692156879, |
| "train/gen_step": 2432.0, |
| "und/clip_ratio/high_mean": 0.000718017738108756, |
| "und/clip_ratio/low_mean": 0.00015644555605831556, |
| "und/clip_ratio/region_mean": 0.0008744632941670716, |
| "und/kl": 0.024760687578236684, |
| "und/loss": 0.001179238304757746 |
| }, |
| { |
| "epoch": 0.49959967974379504, |
| "grad_norm": 0.03500046208500862, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "rewards/correctness_reward_func/mean": 0.71875, |
| "rewards/correctness_reward_func/std": 0.9672207236289978, |
| "rewards/strict_format_reward_func/mean": 0.234375, |
| "rewards/strict_format_reward_func/std": 0.2514837086200714, |
| "step": 39, |
| "time_profile/curr_logprobs_and_update": 0.2616706858971156, |
| "time_profile/old_logps": 33.69548157788813, |
| "time_profile/ref_logps": 33.755788206122816, |
| "time_profile/reward": 0.004642653279006481, |
| "time_profile/text_rollout": 503.46805787831545, |
| "train/gen_step": 2496.0, |
| "und/advantages_abs_mean": 0.399658203125, |
| "und/advantages_max": 1.375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.9375, |
| "und/advantages_std": 0.6456708908081055, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 253.01953125, |
| "und/completion/min_length": 21.0, |
| "und/frac_reward_zero_std": 0.515625, |
| "und/kl": 0.03765736366040073, |
| "und/loss": 0.000907381055640144, |
| "und/prompt/max_length": 313.0, |
| "und/prompt/mean_length": 179.5, |
| "und/prompt/min_length": 46.0, |
| "und/reward": 0.9072265625, |
| "und/reward_std": 1.1414990425109863 |
| }, |
| { |
| "epoch": 0.5124099279423538, |
| "grad_norm": 0.047211844474077225, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "step": 40, |
| "time_profile/curr_logprobs_and_update": 0.2616095836274326, |
| "train/gen_step": 2560.0, |
| "und/clip_ratio/high_mean": 0.00029819617702742107, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.00029819617702742107, |
| "und/kl": 0.03165427554631606, |
| "und/loss": 0.0009896907330357863 |
| }, |
| { |
| "epoch": 0.5252201761409128, |
| "grad_norm": 0.012100051157176495, |
| "learning_rate": 1e-05, |
| "loss": 0.0132, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.171875, |
| "rewards/strict_format_reward_func/std": 0.23935678601264954, |
| "step": 41, |
| "time_profile/curr_logprobs_and_update": 0.26168578446959145, |
| "time_profile/old_logps": 33.68038745317608, |
| "time_profile/ref_logps": 33.777247093617916, |
| "time_profile/reward": 0.004466196522116661, |
| "time_profile/text_rollout": 491.4084973083809, |
| "train/gen_step": 2624.0, |
| "und/advantages_abs_mean": 0.169189453125, |
| "und/advantages_max": 0.4375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.375, |
| "und/advantages_std": 0.20586435496807098, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.802734375, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.109375, |
| "und/kl": 0.33045044860773487, |
| "und/loss": 0.013275288394652307, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 162.0, |
| "und/prompt/min_length": 162.0, |
| "und/reward": 0.1376953125, |
| "und/reward_std": 0.22357389330863953 |
| }, |
| { |
| "epoch": 0.5380304243394716, |
| "grad_norm": 0.009308804757893085, |
| "learning_rate": 1e-05, |
| "loss": 0.0254, |
| "step": 42, |
| "time_profile/curr_logprobs_and_update": 0.26165859241154976, |
| "train/gen_step": 2688.0, |
| "und/clip_ratio/high_mean": 0.00019616381541709416, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.00019616381541709416, |
| "und/kl": 0.6357102408874198, |
| "und/loss": 0.028003237792290747 |
| }, |
| { |
| "epoch": 0.5508406725380304, |
| "grad_norm": 0.055917106568813324, |
| "learning_rate": 1e-05, |
| "loss": 0.0037, |
| "rewards/correctness_reward_func/mean": 0.5625, |
| "rewards/correctness_reward_func/std": 0.9063270092010498, |
| "rewards/strict_format_reward_func/mean": 0.3125, |
| "rewards/strict_format_reward_func/std": 0.24397502839565277, |
| "step": 43, |
| "time_profile/curr_logprobs_and_update": 0.26144762606418226, |
| "time_profile/old_logps": 33.624562768265605, |
| "time_profile/ref_logps": 33.73785715736449, |
| "time_profile/reward": 0.004686887376010418, |
| "time_profile/text_rollout": 454.951701448299, |
| "train/gen_step": 2752.0, |
| "und/advantages_abs_mean": 0.566650390625, |
| "und/advantages_max": 1.5, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.9375, |
| "und/advantages_std": 0.7055483460426331, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.75390625, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.0, |
| "und/kl": 0.09146726290055085, |
| "und/loss": 0.0016110616270452738, |
| "und/prompt/max_length": 57.0, |
| "und/prompt/mean_length": 50.5, |
| "und/prompt/min_length": 44.0, |
| "und/reward": 0.859375, |
| "und/reward_std": 0.9939887523651123 |
| }, |
| { |
| "epoch": 0.5636509207365893, |
| "grad_norm": 0.0613945871591568, |
| "learning_rate": 1e-05, |
| "loss": 0.0074, |
| "step": 44, |
| "time_profile/curr_logprobs_and_update": 0.26146509549289476, |
| "train/gen_step": 2816.0, |
| "und/clip_ratio/high_mean": 0.00215041778574232, |
| "und/clip_ratio/low_mean": 0.00034797564512700774, |
| "und/clip_ratio/region_mean": 0.002498393430869328, |
| "und/kl": 0.0361222850260674, |
| "und/loss": 0.005510351620614529 |
| }, |
| { |
| "epoch": 0.5764611689351481, |
| "grad_norm": 0.009947679936885834, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.28125, |
| "rewards/strict_format_reward_func/std": 0.25, |
| "step": 45, |
| "time_profile/curr_logprobs_and_update": 0.26160548186453525, |
| "time_profile/old_logps": 33.671746659092605, |
| "time_profile/ref_logps": 33.75626815017313, |
| "time_profile/reward": 0.3108100714161992, |
| "time_profile/text_rollout": 472.72174780536443, |
| "train/gen_step": 2880.0, |
| "und/advantages_abs_mean": 0.1962890625, |
| "und/advantages_max": 0.4375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.4375, |
| "und/advantages_std": 0.2217392474412918, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.67578125, |
| "und/completion/min_length": 250.0, |
| "und/frac_reward_zero_std": 0.046875, |
| "und/kl": 0.02999453744996572, |
| "und/loss": 0.0007559580495524187, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 104.5, |
| "und/prompt/min_length": 47.0, |
| "und/reward": 0.255859375, |
| "und/reward_std": 0.25017574429512024 |
| }, |
| { |
| "epoch": 0.589271417133707, |
| "grad_norm": 0.012918645516037941, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "step": 46, |
| "time_profile/curr_logprobs_and_update": 0.2616680209466722, |
| "train/gen_step": 2944.0, |
| "und/clip_ratio/high_mean": 0.0007937824193504639, |
| "und/clip_ratio/low_mean": 4.763719334732741e-05, |
| "und/clip_ratio/region_mean": 0.0008414196126977913, |
| "und/kl": 0.03639418514649151, |
| "und/loss": 0.001037593943193471 |
| }, |
| { |
| "epoch": 0.6020816653322658, |
| "grad_norm": 0.005492714699357748, |
| "learning_rate": 1e-05, |
| "loss": 0.0545, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.140625, |
| "rewards/strict_format_reward_func/std": 0.22658175230026245, |
| "step": 47, |
| "time_profile/curr_logprobs_and_update": 0.2618762481288286, |
| "time_profile/old_logps": 33.7198549490422, |
| "time_profile/ref_logps": 33.79357417766005, |
| "time_profile/reward": 0.004348478280007839, |
| "time_profile/text_rollout": 491.14609040971845, |
| "train/gen_step": 3008.0, |
| "und/advantages_abs_mean": 0.188232421875, |
| "und/advantages_max": 1.9375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.5625, |
| "und/advantages_std": 0.24418380856513977, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.794921875, |
| "und/completion/min_length": 252.0, |
| "und/frac_reward_zero_std": 0.046875, |
| "und/kl": 1.3629103746789042, |
| "und/loss": 0.0005332264117896557, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 162.0, |
| "und/prompt/min_length": 162.0, |
| "und/reward": 0.1552734375, |
| "und/reward_std": 0.26322463154792786 |
| }, |
| { |
| "epoch": 0.6148919135308246, |
| "grad_norm": 0.007346948143094778, |
| "learning_rate": 1e-05, |
| "loss": 0.0133, |
| "step": 48, |
| "time_profile/curr_logprobs_and_update": 0.26197397308715153, |
| "train/gen_step": 3072.0, |
| "und/clip_ratio/high_mean": 4.882812572759576e-05, |
| "und/clip_ratio/low_mean": 7.812499825377017e-05, |
| "und/clip_ratio/region_mean": 0.00012695312398136593, |
| "und/kl": 0.3335349300832604, |
| "und/loss": 0.000871355994604528 |
| }, |
| { |
| "epoch": 0.6277021617293835, |
| "grad_norm": 0.041968513280153275, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "rewards/correctness_reward_func/mean": 0.71875, |
| "rewards/correctness_reward_func/std": 0.9672207236289978, |
| "rewards/strict_format_reward_func/mean": 0.3359375, |
| "rewards/strict_format_reward_func/std": 0.2366211861371994, |
| "step": 49, |
| "time_profile/curr_logprobs_and_update": 0.26143342345312703, |
| "time_profile/old_logps": 33.66150098387152, |
| "time_profile/ref_logps": 33.79388515371829, |
| "time_profile/reward": 0.0042456211522221565, |
| "time_profile/text_rollout": 455.7687773555517, |
| "train/gen_step": 3136.0, |
| "und/advantages_abs_mean": 0.4990234375, |
| "und/advantages_max": 1.5625, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -2.1875, |
| "und/advantages_std": 0.6655394434928894, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.501953125, |
| "und/completion/min_length": 159.0, |
| "und/frac_reward_zero_std": 0.03125, |
| "und/kl": 0.043663310549163725, |
| "und/loss": 0.001001961762085557, |
| "und/prompt/max_length": 57.0, |
| "und/prompt/mean_length": 52.5, |
| "und/prompt/min_length": 48.0, |
| "und/reward": 0.9833984375, |
| "und/reward_std": 1.0895419120788574 |
| }, |
| { |
| "epoch": 0.6405124099279423, |
| "grad_norm": 0.03877226635813713, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "step": 50, |
| "time_profile/curr_logprobs_and_update": 0.26151250462862663, |
| "train/gen_step": 3200.0, |
| "und/clip_ratio/high_mean": 0.0006906339149281848, |
| "und/clip_ratio/low_mean": 0.001115720056986902, |
| "und/clip_ratio/region_mean": 0.0018063539719150867, |
| "und/kl": 0.03958344936108915, |
| "und/loss": 0.0018563272897154093 |
| }, |
| { |
| "epoch": 0.6533226581265013, |
| "grad_norm": 0.040287211537361145, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "rewards/correctness_reward_func/mean": 0.8125, |
| "rewards/correctness_reward_func/std": 0.9900296926498413, |
| "rewards/strict_format_reward_func/mean": 0.359375, |
| "rewards/strict_format_reward_func/std": 0.22658175230026245, |
| "step": 51, |
| "time_profile/curr_logprobs_and_update": 0.2614360564039089, |
| "time_profile/old_logps": 33.677048296667635, |
| "time_profile/ref_logps": 33.734667072072625, |
| "time_profile/reward": 0.00466107577085495, |
| "time_profile/text_rollout": 454.67956179752946, |
| "train/gen_step": 3264.0, |
| "und/advantages_abs_mean": 0.498779296875, |
| "und/advantages_max": 1.8125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -2.1875, |
| "und/advantages_std": 0.669204831123352, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.6875, |
| "und/completion/min_length": 251.0, |
| "und/frac_reward_zero_std": 0.015625, |
| "und/kl": 0.03883673060772708, |
| "und/loss": 0.001454857557376954, |
| "und/prompt/max_length": 55.0, |
| "und/prompt/mean_length": 52.0, |
| "und/prompt/min_length": 49.0, |
| "und/reward": 0.9443359375, |
| "und/reward_std": 1.075581669807434 |
| }, |
| { |
| "epoch": 0.6661329063250601, |
| "grad_norm": 0.038634829223155975, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "step": 52, |
| "time_profile/curr_logprobs_and_update": 0.2613996758009307, |
| "train/gen_step": 3328.0, |
| "und/clip_ratio/high_mean": 0.0003874025678669568, |
| "und/clip_ratio/low_mean": 0.00019797183631453663, |
| "und/clip_ratio/region_mean": 0.0005853744041814934, |
| "und/kl": 0.03153260219551157, |
| "und/loss": 0.0016808376690278237 |
| }, |
| { |
| "epoch": 0.6789431545236189, |
| "grad_norm": 0.05004860833287239, |
| "learning_rate": 1e-05, |
| "loss": 0.0109, |
| "rewards/correctness_reward_func/mean": 0.15625, |
| "rewards/correctness_reward_func/std": 0.5409794449806213, |
| "rewards/strict_format_reward_func/mean": 0.1953125, |
| "rewards/strict_format_reward_func/std": 0.24587368965148926, |
| "step": 53, |
| "time_profile/curr_logprobs_and_update": 0.26163243876362685, |
| "time_profile/old_logps": 33.65926602482796, |
| "time_profile/ref_logps": 33.75773391965777, |
| "time_profile/reward": 0.004499710164964199, |
| "time_profile/text_rollout": 516.6223611980677, |
| "train/gen_step": 3392.0, |
| "und/advantages_abs_mean": 0.2509765625, |
| "und/advantages_max": 2.0625, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.125, |
| "und/advantages_std": 0.5241244435310364, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.7734375, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.5, |
| "und/kl": 0.2721175526821753, |
| "und/loss": 0.054259198752447446, |
| "und/prompt/max_length": 313.0, |
| "und/prompt/mean_length": 234.5, |
| "und/prompt/min_length": 156.0, |
| "und/reward": 0.322265625, |
| "und/reward_std": 0.6323413252830505 |
| }, |
| { |
| "epoch": 0.6917534027221778, |
| "grad_norm": 0.035311710089445114, |
| "learning_rate": 1e-05, |
| "loss": 0.0031, |
| "step": 54, |
| "time_profile/curr_logprobs_and_update": 0.26147235184907913, |
| "train/gen_step": 3456.0, |
| "und/clip_ratio/high_mean": 0.00019807044009212404, |
| "und/clip_ratio/low_mean": 9.089218292501755e-05, |
| "und/clip_ratio/region_mean": 0.0002889626230171416, |
| "und/kl": 0.0721228092443198, |
| "und/loss": 0.002958318164701268 |
| }, |
| { |
| "epoch": 0.7045636509207366, |
| "grad_norm": 0.04633024334907532, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "rewards/correctness_reward_func/mean": 0.90625, |
| "rewards/correctness_reward_func/std": 1.003466248512268, |
| "rewards/strict_format_reward_func/mean": 0.484375, |
| "rewards/strict_format_reward_func/std": 0.08768405020236969, |
| "step": 55, |
| "time_profile/curr_logprobs_and_update": 0.2614114769385196, |
| "time_profile/old_logps": 33.599201523698866, |
| "time_profile/ref_logps": 33.73441112972796, |
| "time_profile/reward": 0.004696068353950977, |
| "time_profile/text_rollout": 453.6825319575146, |
| "train/gen_step": 3520.0, |
| "und/advantages_abs_mean": 0.341064453125, |
| "und/advantages_max": 1.875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -2.1875, |
| "und/advantages_std": 0.6085155010223389, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.72265625, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.40625, |
| "und/kl": 0.03839961695484817, |
| "und/loss": 0.0013346272653507185, |
| "und/prompt/max_length": 47.0, |
| "und/prompt/mean_length": 45.5, |
| "und/prompt/min_length": 44.0, |
| "und/reward": 1.3662109375, |
| "und/reward_std": 1.0347063541412354 |
| }, |
| { |
| "epoch": 0.7173738991192954, |
| "grad_norm": 0.047499582171440125, |
| "learning_rate": 1e-05, |
| "loss": 0.0034, |
| "step": 56, |
| "time_profile/curr_logprobs_and_update": 0.2612421384692425, |
| "train/gen_step": 3584.0, |
| "und/clip_ratio/high_mean": 0.0003667974451673217, |
| "und/clip_ratio/low_mean": 0.00013469014447764494, |
| "und/clip_ratio/region_mean": 0.0005014875860069878, |
| "und/kl": 0.08546131956973113, |
| "und/loss": 0.00697715762282769 |
| }, |
| { |
| "epoch": 0.7301841473178543, |
| "grad_norm": 0.014783147722482681, |
| "learning_rate": 1e-05, |
| "loss": 0.004, |
| "rewards/correctness_reward_func/mean": 0.09375, |
| "rewards/correctness_reward_func/std": 0.42608407139778137, |
| "rewards/strict_format_reward_func/mean": 0.296875, |
| "rewards/strict_format_reward_func/std": 0.24750742316246033, |
| "step": 57, |
| "time_profile/curr_logprobs_and_update": 0.2614298419211991, |
| "time_profile/old_logps": 33.64691087603569, |
| "time_profile/ref_logps": 33.739184183999896, |
| "time_profile/reward": 0.004393537528812885, |
| "time_profile/text_rollout": 454.8285795925185, |
| "train/gen_step": 3648.0, |
| "und/advantages_abs_mean": 0.20654296875, |
| "und/advantages_max": 1.8125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.9375, |
| "und/advantages_std": 0.3522537350654602, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.716796875, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.21875, |
| "und/kl": 0.10082034709921572, |
| "und/loss": 0.0015968736425975294, |
| "und/prompt/max_length": 56.0, |
| "und/prompt/mean_length": 51.0, |
| "und/prompt/min_length": 46.0, |
| "und/reward": 0.3818359375, |
| "und/reward_std": 0.42684176564216614 |
| }, |
| { |
| "epoch": 0.7429943955164131, |
| "grad_norm": 0.014119211584329605, |
| "learning_rate": 1e-05, |
| "loss": 0.0117, |
| "step": 58, |
| "time_profile/curr_logprobs_and_update": 0.26134903807542287, |
| "train/gen_step": 3712.0, |
| "und/clip_ratio/high_mean": 4.882812572759576e-05, |
| "und/clip_ratio/low_mean": 0.00023804241573088802, |
| "und/clip_ratio/region_mean": 0.0002868705414584838, |
| "und/kl": 0.2931558708951343, |
| "und/loss": 0.001128198406345149 |
| }, |
| { |
| "epoch": 0.755804643714972, |
| "grad_norm": 0.03397494554519653, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "rewards/correctness_reward_func/mean": 0.15625, |
| "rewards/correctness_reward_func/std": 0.5409794449806213, |
| "rewards/strict_format_reward_func/mean": 0.484375, |
| "rewards/strict_format_reward_func/std": 0.08768405020236969, |
| "step": 59, |
| "time_profile/curr_logprobs_and_update": 0.2614648619637592, |
| "time_profile/old_logps": 33.627349744550884, |
| "time_profile/ref_logps": 33.70189340412617, |
| "time_profile/reward": 0.00450173020362854, |
| "time_profile/text_rollout": 454.01288072485477, |
| "train/gen_step": 3776.0, |
| "und/advantages_abs_mean": 0.373046875, |
| "und/advantages_max": 1.8125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.6875, |
| "und/advantages_std": 0.6158081293106079, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.609375, |
| "und/completion/min_length": 251.0, |
| "und/frac_reward_zero_std": 0.453125, |
| "und/kl": 0.07084466355445329, |
| "und/loss": 0.002370536963894665, |
| "und/prompt/max_length": 47.0, |
| "und/prompt/mean_length": 46.0, |
| "und/prompt/min_length": 45.0, |
| "und/reward": 0.7958984375, |
| "und/reward_std": 0.739467978477478 |
| }, |
| { |
| "epoch": 0.7686148919135308, |
| "grad_norm": 0.046682216227054596, |
| "learning_rate": 1e-05, |
| "loss": 0.0041, |
| "step": 60, |
| "time_profile/curr_logprobs_and_update": 0.26129540299007203, |
| "train/gen_step": 3840.0, |
| "und/clip_ratio/high_mean": 0.000178795224201167, |
| "und/clip_ratio/low_mean": 0.0001412087440257892, |
| "und/clip_ratio/region_mean": 0.0003200039645889774, |
| "und/kl": 0.10260857833782211, |
| "und/loss": 0.0027817148465487662 |
| }, |
| { |
| "epoch": 0.7814251401120896, |
| "grad_norm": 0.014530722051858902, |
| "learning_rate": 1e-05, |
| "loss": 0.0184, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.1953125, |
| "rewards/strict_format_reward_func/std": 0.24587368965148926, |
| "step": 61, |
| "time_profile/curr_logprobs_and_update": 0.2614762183802668, |
| "time_profile/old_logps": 33.65403878968209, |
| "time_profile/ref_logps": 33.731373984366655, |
| "time_profile/reward": 0.004584863781929016, |
| "time_profile/text_rollout": 472.9927922273055, |
| "train/gen_step": 3904.0, |
| "und/advantages_abs_mean": 0.203125, |
| "und/advantages_max": 1.5, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.5, |
| "und/advantages_std": 0.23512499034404755, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.74609375, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.015625, |
| "und/kl": 0.4595181476797734, |
| "und/loss": 0.0014087316812947392, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 106.0, |
| "und/prompt/min_length": 50.0, |
| "und/reward": 0.20703125, |
| "und/reward_std": 0.25815349817276 |
| }, |
| { |
| "epoch": 0.7942353883106485, |
| "grad_norm": 0.014059546403586864, |
| "learning_rate": 1e-05, |
| "loss": 0.0132, |
| "step": 62, |
| "time_profile/curr_logprobs_and_update": 0.2614812095707748, |
| "train/gen_step": 3968.0, |
| "und/clip_ratio/high_mean": 0.0003522909028106369, |
| "und/clip_ratio/low_mean": 0.00013392856999416836, |
| "und/clip_ratio/region_mean": 0.00048621947280480526, |
| "und/kl": 0.32975731499755057, |
| "und/loss": 0.0033024766016751528 |
| }, |
| { |
| "epoch": 0.8070456365092074, |
| "grad_norm": 0.013789220713078976, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.2890625, |
| "rewards/strict_format_reward_func/std": 0.24888142943382263, |
| "step": 63, |
| "time_profile/curr_logprobs_and_update": 0.26154088985640556, |
| "time_profile/old_logps": 33.65780791267753, |
| "time_profile/ref_logps": 33.74037241283804, |
| "time_profile/reward": 0.003031497821211815, |
| "time_profile/text_rollout": 454.97051773872226, |
| "train/gen_step": 4032.0, |
| "und/advantages_abs_mean": 0.210693359375, |
| "und/advantages_max": 0.375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.4375, |
| "und/advantages_std": 0.22973118722438812, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.74609375, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.046875, |
| "und/kl": 0.033402759159798734, |
| "und/loss": 0.0011463420676136593, |
| "und/prompt/max_length": 52.0, |
| "und/prompt/mean_length": 50.5, |
| "und/prompt/min_length": 49.0, |
| "und/reward": 0.2783203125, |
| "und/reward_std": 0.24863366782665253 |
| }, |
| { |
| "epoch": 0.8198558847077662, |
| "grad_norm": 0.015820274129509926, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "step": 64, |
| "time_profile/curr_logprobs_and_update": 0.2614853223931277, |
| "train/gen_step": 4096.0, |
| "und/clip_ratio/high_mean": 0.0008117913530441001, |
| "und/clip_ratio/low_mean": 0.00011440205707913265, |
| "und/clip_ratio/region_mean": 0.0009261934137612116, |
| "und/kl": 0.03982456024095882, |
| "und/loss": 0.0010868344670598162 |
| }, |
| { |
| "epoch": 0.8326661329063251, |
| "grad_norm": 0.041560448706150055, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "rewards/correctness_reward_func/mean": 0.28125, |
| "rewards/correctness_reward_func/std": 0.7007648944854736, |
| "rewards/strict_format_reward_func/mean": 0.21875, |
| "rewards/strict_format_reward_func/std": 0.25, |
| "step": 65, |
| "time_profile/curr_logprobs_and_update": 0.26148034693324007, |
| "time_profile/old_logps": 33.658358512446284, |
| "time_profile/ref_logps": 33.74039072729647, |
| "time_profile/reward": 0.0044120242819190025, |
| "time_profile/text_rollout": 473.54834595508873, |
| "train/gen_step": 4160.0, |
| "und/advantages_abs_mean": 0.516845703125, |
| "und/advantages_max": 2.0625, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.6875, |
| "und/advantages_std": 0.7237750291824341, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.75390625, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.03125, |
| "und/kl": 0.061325576913077384, |
| "und/loss": 0.0018601922784000635, |
| "und/prompt/max_length": 156.0, |
| "und/prompt/mean_length": 109.5, |
| "und/prompt/min_length": 63.0, |
| "und/reward": 0.5234375, |
| "und/reward_std": 0.8563321828842163 |
| }, |
| { |
| "epoch": 0.8454763811048839, |
| "grad_norm": 0.040835775434970856, |
| "learning_rate": 1e-05, |
| "loss": 0.0047, |
| "step": 66, |
| "time_profile/curr_logprobs_and_update": 0.2613908527127933, |
| "train/gen_step": 4224.0, |
| "und/clip_ratio/high_mean": 0.0002187356585636735, |
| "und/clip_ratio/low_mean": 0.0002555298342485912, |
| "und/clip_ratio/region_mean": 0.0004742654928122647, |
| "und/kl": 0.11611133666156093, |
| "und/loss": 0.0033431433839723468 |
| }, |
| { |
| "epoch": 0.8582866293034428, |
| "grad_norm": 0.04072283208370209, |
| "learning_rate": 1e-05, |
| "loss": 0.0042, |
| "rewards/correctness_reward_func/mean": 0.875, |
| "rewards/correctness_reward_func/std": 1.0, |
| "rewards/strict_format_reward_func/mean": 0.3828125, |
| "rewards/strict_format_reward_func/std": 0.21347814798355103, |
| "step": 67, |
| "time_profile/curr_logprobs_and_update": 0.26157006529683713, |
| "time_profile/old_logps": 33.639826339669526, |
| "time_profile/ref_logps": 33.74620936065912, |
| "time_profile/reward": 0.004473298788070679, |
| "time_profile/text_rollout": 454.7873819489032, |
| "train/gen_step": 4288.0, |
| "und/advantages_abs_mean": 0.275390625, |
| "und/advantages_max": 0.9375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -2.1875, |
| "und/advantages_std": 0.4825730323791504, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.744140625, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.203125, |
| "und/kl": 0.10618040141707752, |
| "und/loss": 0.002721756522078067, |
| "und/prompt/max_length": 53.0, |
| "und/prompt/mean_length": 49.5, |
| "und/prompt/min_length": 46.0, |
| "und/reward": 1.283203125, |
| "und/reward_std": 1.1394332647323608 |
| }, |
| { |
| "epoch": 0.8710968775020016, |
| "grad_norm": 0.023537058383226395, |
| "learning_rate": 1e-05, |
| "loss": 0.0019, |
| "step": 68, |
| "time_profile/curr_logprobs_and_update": 0.2615145814634161, |
| "train/gen_step": 4352.0, |
| "und/clip_ratio/high_mean": 0.0006931504431122448, |
| "und/clip_ratio/low_mean": 0.000173399384948425, |
| "und/clip_ratio/region_mean": 0.000866549824422691, |
| "und/kl": 0.05065767532505561, |
| "und/loss": 0.0008179922006092966 |
| }, |
| { |
| "epoch": 0.8839071257005604, |
| "grad_norm": 0.00545235862955451, |
| "learning_rate": 1e-05, |
| "loss": 0.0019, |
| "rewards/correctness_reward_func/mean": 0.03125, |
| "rewards/correctness_reward_func/std": 0.25, |
| "rewards/strict_format_reward_func/mean": 0.0703125, |
| "rewards/strict_format_reward_func/std": 0.1751912236213684, |
| "step": 69, |
| "time_profile/curr_logprobs_and_update": 0.261740165471565, |
| "time_profile/old_logps": 33.70919596590102, |
| "time_profile/ref_logps": 33.77206904441118, |
| "time_profile/reward": 0.004499014467000961, |
| "time_profile/text_rollout": 501.05089360289276, |
| "train/gen_step": 4416.0, |
| "und/advantages_abs_mean": 0.09765625, |
| "und/advantages_max": 2.125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.4375, |
| "und/advantages_std": 0.18104934692382812, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.92578125, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.53125, |
| "und/kl": 0.04702442421694286, |
| "und/loss": 0.0023319427573369467, |
| "und/prompt/max_length": 313.0, |
| "und/prompt/mean_length": 185.5, |
| "und/prompt/min_length": 58.0, |
| "und/reward": 0.083984375, |
| "und/reward_std": 0.21164102852344513 |
| }, |
| { |
| "epoch": 0.8967173738991193, |
| "grad_norm": 0.015413722954690456, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "step": 70, |
| "time_profile/curr_logprobs_and_update": 0.26160615020489786, |
| "train/gen_step": 4480.0, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 9.790100375539623e-05, |
| "und/clip_ratio/region_mean": 9.790100375539623e-05, |
| "und/kl": 0.0741199756594142, |
| "und/loss": 0.002338052549376357 |
| }, |
| { |
| "epoch": 0.9095276220976781, |
| "grad_norm": 0.013947661966085434, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.09375, |
| "rewards/strict_format_reward_func/std": 0.19669894874095917, |
| "step": 71, |
| "time_profile/curr_logprobs_and_update": 0.26171521554351784, |
| "time_profile/old_logps": 33.66377367079258, |
| "time_profile/ref_logps": 33.751427008770406, |
| "time_profile/reward": 0.004377297125756741, |
| "time_profile/text_rollout": 500.3317451989278, |
| "train/gen_step": 4544.0, |
| "und/advantages_abs_mean": 0.107666015625, |
| "und/advantages_max": 0.4375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.4375, |
| "und/advantages_std": 0.16422295570373535, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.857421875, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.5, |
| "und/kl": 0.04332471443194663, |
| "und/loss": 0.0016425758730918005, |
| "und/prompt/max_length": 313.0, |
| "und/prompt/mean_length": 184.0, |
| "und/prompt/min_length": 55.0, |
| "und/reward": 0.1005859375, |
| "und/reward_std": 0.2006341516971588 |
| }, |
| { |
| "epoch": 0.922337870296237, |
| "grad_norm": 0.012040519155561924, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "step": 72, |
| "time_profile/curr_logprobs_and_update": 0.2617163411487127, |
| "train/gen_step": 4608.0, |
| "und/clip_ratio/high_mean": 0.00010259100599796511, |
| "und/clip_ratio/low_mean": 0.00037950047408230603, |
| "und/clip_ratio/region_mean": 0.00048209148008027114, |
| "und/kl": 0.03941097212373279, |
| "und/loss": 0.0017510293993154846 |
| }, |
| { |
| "epoch": 0.9351481184947958, |
| "grad_norm": 0.025353815406560898, |
| "learning_rate": 1e-05, |
| "loss": 0.0082, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.234375, |
| "rewards/strict_format_reward_func/std": 0.2514837086200714, |
| "step": 73, |
| "time_profile/curr_logprobs_and_update": 0.26159169392485637, |
| "time_profile/old_logps": 33.65948408655822, |
| "time_profile/ref_logps": 33.75273651909083, |
| "time_profile/reward": 0.004401316866278648, |
| "time_profile/text_rollout": 487.3236538115889, |
| "train/gen_step": 4672.0, |
| "und/advantages_abs_mean": 0.070556640625, |
| "und/advantages_max": 2.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.6875, |
| "und/advantages_std": 0.2424243539571762, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.818359375, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.703125, |
| "und/kl": 0.20393660698027816, |
| "und/loss": 0.004929899203489185, |
| "und/prompt/max_length": 247.0, |
| "und/prompt/mean_length": 148.0, |
| "und/prompt/min_length": 49.0, |
| "und/reward": 0.2666015625, |
| "und/reward_std": 0.35626593232154846 |
| }, |
| { |
| "epoch": 0.9479583666933546, |
| "grad_norm": 0.013712815009057522, |
| "learning_rate": 1e-05, |
| "loss": 0.0149, |
| "step": 74, |
| "time_profile/curr_logprobs_and_update": 0.26143009605584666, |
| "train/gen_step": 4736.0, |
| "und/clip_ratio/high_mean": 0.00025846589414868504, |
| "und/clip_ratio/low_mean": 5.278716344037093e-05, |
| "und/clip_ratio/region_mean": 0.00031125305758905597, |
| "und/kl": 0.36927700173691846, |
| "und/loss": 0.0025039048924071494 |
| }, |
| { |
| "epoch": 0.9607686148919136, |
| "grad_norm": 0.009972140192985535, |
| "learning_rate": 1e-05, |
| "loss": 0.0136, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.21875, |
| "rewards/strict_format_reward_func/std": 0.25, |
| "step": 75, |
| "time_profile/curr_logprobs_and_update": 0.26166287070373073, |
| "time_profile/old_logps": 33.67625562660396, |
| "time_profile/ref_logps": 33.7687153192237, |
| "time_profile/reward": 0.0043565016239881516, |
| "time_profile/text_rollout": 473.3151173014194, |
| "train/gen_step": 4800.0, |
| "und/advantages_abs_mean": 0.205810546875, |
| "und/advantages_max": 0.4375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.375, |
| "und/advantages_std": 0.22705358266830444, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.791015625, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.015625, |
| "und/kl": 0.33891669298463967, |
| "und/loss": 0.0008663814514875412, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 107.0, |
| "und/prompt/min_length": 52.0, |
| "und/reward": 0.1904296875, |
| "und/reward_std": 0.24303650856018066 |
| }, |
| { |
| "epoch": 0.9735788630904724, |
| "grad_norm": 0.009620222263038158, |
| "learning_rate": 1e-05, |
| "loss": 0.0059, |
| "step": 76, |
| "time_profile/curr_logprobs_and_update": 0.2629899470921373, |
| "train/gen_step": 4864.0, |
| "und/clip_ratio/high_mean": 0.00046749321700190194, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.00046749321700190194, |
| "und/kl": 0.1477012149625807, |
| "und/loss": 0.0013083760859444737 |
| }, |
| { |
| "epoch": 0.9863891112890312, |
| "grad_norm": 0.010119947604835033, |
| "learning_rate": 1e-05, |
| "loss": 0.0234, |
| "rewards/correctness_reward_func/mean": 0.03125, |
| "rewards/correctness_reward_func/std": 0.25, |
| "rewards/strict_format_reward_func/mean": 0.2578125, |
| "rewards/strict_format_reward_func/std": 0.25185325741767883, |
| "step": 77, |
| "time_profile/curr_logprobs_and_update": 0.2616106412606314, |
| "time_profile/old_logps": 33.68558882549405, |
| "time_profile/ref_logps": 33.77748038619757, |
| "time_profile/reward": 0.0041413139551877975, |
| "time_profile/text_rollout": 473.13247388228774, |
| "train/gen_step": 4928.0, |
| "und/advantages_abs_mean": 0.216064453125, |
| "und/advantages_max": 2.125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.5, |
| "und/advantages_std": 0.2934376895427704, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.806640625, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.0625, |
| "und/kl": 0.5857936546090059, |
| "und/loss": 0.0010301140270740916, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 106.5, |
| "und/prompt/min_length": 51.0, |
| "und/reward": 0.220703125, |
| "und/reward_std": 0.31764960289001465 |
| }, |
| { |
| "epoch": 0.9991993594875901, |
| "grad_norm": 0.012176643125712872, |
| "learning_rate": 1e-05, |
| "loss": 0.0303, |
| "step": 78, |
| "time_profile/curr_logprobs_and_update": 0.2613839999830816, |
| "train/gen_step": 4992.0, |
| "und/clip_ratio/high_mean": 0.00017109220061684027, |
| "und/clip_ratio/low_mean": 0.000871925825776998, |
| "und/clip_ratio/region_mean": 0.0010430180263938382, |
| "und/kl": 0.7574968582484871, |
| "und/loss": 0.0010476876539087243 |
| }, |
| { |
| "epoch": 1.0, |
| "grad_norm": 0.010883414186537266, |
| "learning_rate": 1e-05, |
| "loss": 0.001, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.2890625, |
| "rewards/strict_format_reward_func/std": 0.24888142943382263, |
| "step": 79, |
| "time_profile/curr_logprobs_and_update": 0.26290554786100984, |
| "time_profile/old_logps": 33.66828224621713, |
| "time_profile/ref_logps": 33.729235981591046, |
| "time_profile/reward": 0.004360824823379517, |
| "time_profile/text_rollout": 456.6069682110101, |
| "train/gen_step": 4996.0, |
| "und/advantages_abs_mean": 0.123779296875, |
| "und/advantages_max": 0.4375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.4375, |
| "und/advantages_std": 0.17608344554901123, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.671875, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.3125, |
| "und/kl": 0.018394303042441607, |
| "und/loss": 0.12604861333966255, |
| "und/prompt/max_length": 64.0, |
| "und/prompt/mean_length": 57.0, |
| "und/prompt/min_length": 50.0, |
| "und/reward": 0.3056640625, |
| "und/reward_std": 0.2439626157283783 |
| }, |
| { |
| "epoch": 1.0128102481985588, |
| "grad_norm": 0.011202736757695675, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "step": 80, |
| "time_profile/curr_logprobs_and_update": 0.26144698687130585, |
| "train/gen_step": 5060.0, |
| "und/clip_ratio/high_mean": 0.00020480685998336412, |
| "und/clip_ratio/low_mean": 0.0010958394632325508, |
| "und/clip_ratio/region_mean": 0.001300646319577936, |
| "und/kl": 0.06380279999575578, |
| "und/loss": 0.001128781703300774 |
| }, |
| { |
| "epoch": 1.0256204963971176, |
| "grad_norm": 0.011037485674023628, |
| "learning_rate": 1e-05, |
| "loss": -0.0013, |
| "rewards/correctness_reward_func/mean": 0.96875, |
| "rewards/correctness_reward_func/std": 1.0074130296707153, |
| "rewards/strict_format_reward_func/mean": 0.296875, |
| "rewards/strict_format_reward_func/std": 0.24750742316246033, |
| "step": 81, |
| "time_profile/curr_logprobs_and_update": 0.2614856250002049, |
| "time_profile/old_logps": 33.64920091070235, |
| "time_profile/ref_logps": 33.70459945779294, |
| "time_profile/reward": 0.004259305074810982, |
| "time_profile/text_rollout": 454.86890551261604, |
| "train/gen_step": 5124.0, |
| "und/advantages_abs_mean": 0.21484375, |
| "und/advantages_max": 0.9375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -2.1875, |
| "und/advantages_std": 0.4081484079360962, |
| "und/clip_ratio/high_mean": 0.0003931849460059311, |
| "und/clip_ratio/low_mean": 0.0013027411841903813, |
| "und/clip_ratio/region_mean": 0.0016959261301963124, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.779296875, |
| "und/completion/min_length": 222.0, |
| "und/frac_reward_zero_std": 0.328125, |
| "und/kl": 0.06465533758455422, |
| "und/loss": -0.005637970802126802, |
| "und/prompt/max_length": 58.0, |
| "und/prompt/mean_length": 51.0, |
| "und/prompt/min_length": 44.0, |
| "und/reward": 1.2578125, |
| "und/reward_std": 1.1828593015670776 |
| }, |
| { |
| "epoch": 1.0384307445956766, |
| "grad_norm": 0.012660922482609749, |
| "learning_rate": 1e-05, |
| "loss": 0.0052, |
| "step": 82, |
| "time_profile/curr_logprobs_and_update": 0.2613754039048217, |
| "train/gen_step": 5188.0, |
| "und/clip_ratio/high_mean": 0.00011789133350248449, |
| "und/clip_ratio/low_mean": 0.0001570866115798708, |
| "und/clip_ratio/region_mean": 0.0002749779450823553, |
| "und/kl": 0.13132090101134963, |
| "und/loss": 0.0017235139853255532 |
| }, |
| { |
| "epoch": 1.0512409927942354, |
| "grad_norm": 0.04931657388806343, |
| "learning_rate": 1e-05, |
| "loss": 0.0348, |
| "rewards/correctness_reward_func/mean": 0.3125, |
| "rewards/correctness_reward_func/std": 0.7319250702857971, |
| "rewards/strict_format_reward_func/mean": 0.390625, |
| "rewards/strict_format_reward_func/std": 0.2083333432674408, |
| "step": 83, |
| "time_profile/curr_logprobs_and_update": 0.2614861790498253, |
| "time_profile/old_logps": 33.64623137563467, |
| "time_profile/ref_logps": 33.70282917190343, |
| "time_profile/reward": 0.004459716379642487, |
| "time_profile/text_rollout": 454.369918580167, |
| "train/gen_step": 5252.0, |
| "und/advantages_abs_mean": 0.470703125, |
| "und/advantages_max": 1.75, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.4375, |
| "und/advantages_std": 0.6308677792549133, |
| "und/clip_ratio/high_mean": 0.00037126274764887057, |
| "und/clip_ratio/low_mean": 0.0008432684480794705, |
| "und/clip_ratio/region_mean": 0.0012145311957283411, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.71875, |
| "und/completion/min_length": 220.0, |
| "und/frac_reward_zero_std": 0.046875, |
| "und/kl": 0.8189952824468492, |
| "und/loss": 0.002741719154414568, |
| "und/prompt/max_length": 51.0, |
| "und/prompt/mean_length": 47.5, |
| "und/prompt/min_length": 44.0, |
| "und/reward": 0.689453125, |
| "und/reward_std": 0.7827932238578796 |
| }, |
| { |
| "epoch": 1.0640512409927942, |
| "grad_norm": 0.0498473197221756, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "step": 84, |
| "time_profile/curr_logprobs_and_update": 0.26133355250931345, |
| "train/gen_step": 5316.0, |
| "und/clip_ratio/high_mean": 0.0004264039162080735, |
| "und/clip_ratio/low_mean": 0.0006009412281855475, |
| "und/clip_ratio/region_mean": 0.0010273451480315998, |
| "und/kl": 0.060061977404984646, |
| "und/loss": 0.002192927919622889 |
| }, |
| { |
| "epoch": 1.076861489191353, |
| "grad_norm": 0.037328220903873444, |
| "learning_rate": 1e-05, |
| "loss": 0.0067, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.4921875, |
| "rewards/strict_format_reward_func/std": 0.0625, |
| "step": 85, |
| "time_profile/curr_logprobs_and_update": 0.26134839945007116, |
| "time_profile/old_logps": 33.615224280394614, |
| "time_profile/ref_logps": 33.697587930597365, |
| "time_profile/reward": 0.004523534327745438, |
| "time_profile/text_rollout": 454.07083107624203, |
| "train/gen_step": 5380.0, |
| "und/advantages_abs_mean": 0.103759765625, |
| "und/advantages_max": 1.8125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.6875, |
| "und/advantages_std": 0.28764939308166504, |
| "und/clip_ratio/high_mean": 0.0008836273773340508, |
| "und/clip_ratio/low_mean": 0.001003391091217054, |
| "und/clip_ratio/region_mean": 0.0018870184685511049, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.63671875, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.625, |
| "und/kl": 0.05825820894096978, |
| "und/loss": 0.002590392931324459, |
| "und/prompt/max_length": 47.0, |
| "und/prompt/mean_length": 45.0, |
| "und/prompt/min_length": 43.0, |
| "und/reward": 0.5224609375, |
| "und/reward_std": 0.30963554978370667 |
| }, |
| { |
| "epoch": 1.0896717373899119, |
| "grad_norm": 0.029800761491060257, |
| "learning_rate": 1e-05, |
| "loss": 0.0081, |
| "step": 86, |
| "time_profile/curr_logprobs_and_update": 0.2612544225266902, |
| "train/gen_step": 5444.0, |
| "und/clip_ratio/high_mean": 9.385726662003435e-05, |
| "und/clip_ratio/low_mean": 9.646531907492317e-05, |
| "und/clip_ratio/region_mean": 0.00019032258569495752, |
| "und/kl": 0.20097231600084342, |
| "und/loss": 0.0032604012124011206 |
| }, |
| { |
| "epoch": 1.1024819855884709, |
| "grad_norm": 0.03139861673116684, |
| "learning_rate": 1e-05, |
| "loss": 0.0314, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.09375, |
| "rewards/strict_format_reward_func/std": 0.19669894874095917, |
| "step": 87, |
| "time_profile/curr_logprobs_and_update": 0.26139444966975134, |
| "time_profile/old_logps": 33.700514083728194, |
| "time_profile/ref_logps": 33.78526249341667, |
| "time_profile/reward": 0.004218182526528835, |
| "time_profile/text_rollout": 518.5401958124712, |
| "train/gen_step": 5508.0, |
| "und/advantages_abs_mean": 0.0966796875, |
| "und/advantages_max": 0.4375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.3125, |
| "und/advantages_std": 0.1556188315153122, |
| "und/clip_ratio/high_mean": 0.00026991102640749887, |
| "und/clip_ratio/low_mean": 0.0002991936562466435, |
| "und/clip_ratio/region_mean": 0.0005691046826541424, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.720703125, |
| "und/completion/min_length": 159.0, |
| "und/frac_reward_zero_std": 0.5, |
| "und/kl": 0.7756888958901982, |
| "und/loss": 0.01109763615556858, |
| "und/prompt/max_length": 313.0, |
| "und/prompt/mean_length": 237.5, |
| "und/prompt/min_length": 162.0, |
| "und/reward": 0.080078125, |
| "und/reward_std": 0.18355479836463928 |
| }, |
| { |
| "epoch": 1.1152922337870297, |
| "grad_norm": 0.007225779816508293, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "step": 88, |
| "time_profile/curr_logprobs_and_update": 0.2618225736368913, |
| "train/gen_step": 5572.0, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/kl": 0.05710822065520915, |
| "und/loss": 0.0013941282181804127 |
| }, |
| { |
| "epoch": 1.1281024819855885, |
| "grad_norm": 0.006840178277343512, |
| "learning_rate": 1e-05, |
| "loss": 0.0449, |
| "rewards/correctness_reward_func/mean": 0.28125, |
| "rewards/correctness_reward_func/std": 0.7007648944854736, |
| "rewards/strict_format_reward_func/mean": 0.2109375, |
| "rewards/strict_format_reward_func/std": 0.24888142943382263, |
| "step": 89, |
| "time_profile/curr_logprobs_and_update": 0.2619360370154027, |
| "time_profile/old_logps": 33.66616447176784, |
| "time_profile/ref_logps": 33.762949594296515, |
| "time_profile/reward": 0.004555100575089455, |
| "time_profile/text_rollout": 499.7487023388967, |
| "train/gen_step": 5636.0, |
| "und/advantages_abs_mean": 0.455078125, |
| "und/advantages_max": 1.8125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.9375, |
| "und/advantages_std": 0.6984922289848328, |
| "und/clip_ratio/high_mean": 0.00011725750300684012, |
| "und/clip_ratio/low_mean": 0.0001583614903211128, |
| "und/clip_ratio/region_mean": 0.0002756189933279529, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.771484375, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.46875, |
| "und/kl": 1.232714807476441, |
| "und/loss": -0.006696377290722921, |
| "und/prompt/max_length": 313.0, |
| "und/prompt/mean_length": 180.0, |
| "und/prompt/min_length": 47.0, |
| "und/reward": 0.734375, |
| "und/reward_std": 1.0485988855361938 |
| }, |
| { |
| "epoch": 1.1409127301841473, |
| "grad_norm": 0.006123845931142569, |
| "learning_rate": 1e-05, |
| "loss": 0.004, |
| "step": 90, |
| "time_profile/curr_logprobs_and_update": 0.26148606633068994, |
| "train/gen_step": 5700.0, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.00026530873219599016, |
| "und/clip_ratio/region_mean": 0.00026530873219599016, |
| "und/kl": 0.09945900924503803, |
| "und/loss": 0.004926550779600802 |
| }, |
| { |
| "epoch": 1.153722978382706, |
| "grad_norm": 0.009329823777079582, |
| "learning_rate": 1e-05, |
| "loss": 0.0647, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.3046875, |
| "rewards/strict_format_reward_func/std": 0.24587368965148926, |
| "step": 91, |
| "time_profile/curr_logprobs_and_update": 0.26152575001469813, |
| "time_profile/old_logps": 33.65729128010571, |
| "time_profile/ref_logps": 33.73677711561322, |
| "time_profile/reward": 0.004335583187639713, |
| "time_profile/text_rollout": 472.53139427211136, |
| "train/gen_step": 5764.0, |
| "und/advantages_abs_mean": 0.19921875, |
| "und/advantages_max": 1.8125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.6875, |
| "und/advantages_std": 0.35682472586631775, |
| "und/clip_ratio/high_mean": 0.0003500454331515357, |
| "und/clip_ratio/low_mean": 0.00026731862817541696, |
| "und/clip_ratio/region_mean": 0.0006173640649649315, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.72265625, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.265625, |
| "und/kl": 1.6728466484637465, |
| "und/loss": 0.006491393034089299, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 104.0, |
| "und/prompt/min_length": 46.0, |
| "und/reward": 0.359375, |
| "und/reward_std": 0.4551832377910614 |
| }, |
| { |
| "epoch": 1.1665332265812651, |
| "grad_norm": 3.300291061401367, |
| "learning_rate": 1e-05, |
| "loss": 0.021, |
| "step": 92, |
| "time_profile/curr_logprobs_and_update": 0.2615345583180897, |
| "train/gen_step": 5828.0, |
| "und/clip_ratio/high_mean": 0.00015658686243114062, |
| "und/clip_ratio/low_mean": 0.000412728051742306, |
| "und/clip_ratio/region_mean": 0.0005693149105354678, |
| "und/kl": 0.5270771795330802, |
| "und/loss": 0.021786472902746823 |
| }, |
| { |
| "epoch": 1.179343474779824, |
| "grad_norm": 0.04499354213476181, |
| "learning_rate": 1e-05, |
| "loss": 0.0101, |
| "rewards/correctness_reward_func/mean": 0.78125, |
| "rewards/correctness_reward_func/std": 0.983494758605957, |
| "rewards/strict_format_reward_func/mean": 0.359375, |
| "rewards/strict_format_reward_func/std": 0.22658175230026245, |
| "step": 93, |
| "time_profile/curr_logprobs_and_update": 0.2617357307317434, |
| "time_profile/old_logps": 33.68263782840222, |
| "time_profile/ref_logps": 33.77571779489517, |
| "time_profile/reward": 0.004069678485393524, |
| "time_profile/text_rollout": 456.24606967996806, |
| "train/gen_step": 5892.0, |
| "und/advantages_abs_mean": 0.408447265625, |
| "und/advantages_max": 0.875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -2.1875, |
| "und/advantages_std": 0.5925272703170776, |
| "und/clip_ratio/high_mean": 0.0006427463522413746, |
| "und/clip_ratio/low_mean": 0.0003597673749027308, |
| "und/clip_ratio/region_mean": 0.0010025137307820842, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.634765625, |
| "und/completion/min_length": 173.0, |
| "und/frac_reward_zero_std": 0.078125, |
| "und/kl": 0.16817912326951046, |
| "und/loss": 0.0022755718414586568, |
| "und/prompt/max_length": 65.0, |
| "und/prompt/mean_length": 54.0, |
| "und/prompt/min_length": 43.0, |
| "und/reward": 1.1474609375, |
| "und/reward_std": 1.1275041103363037 |
| }, |
| { |
| "epoch": 1.1921537229783827, |
| "grad_norm": 0.06060933321714401, |
| "learning_rate": 1e-05, |
| "loss": 0.0042, |
| "step": 94, |
| "time_profile/curr_logprobs_and_update": 0.26131482362688985, |
| "train/gen_step": 5956.0, |
| "und/clip_ratio/high_mean": 0.0019653168055810966, |
| "und/clip_ratio/low_mean": 0.00016475993470521644, |
| "und/clip_ratio/region_mean": 0.002130076743924292, |
| "und/kl": 0.10070258447376546, |
| "und/loss": 0.005654156702803448 |
| }, |
| { |
| "epoch": 1.2049639711769415, |
| "grad_norm": 0.03084016963839531, |
| "learning_rate": 1e-05, |
| "loss": 0.0413, |
| "rewards/correctness_reward_func/mean": 0.03125, |
| "rewards/correctness_reward_func/std": 0.25, |
| "rewards/strict_format_reward_func/mean": 0.265625, |
| "rewards/strict_format_reward_func/std": 0.2514837086200714, |
| "step": 95, |
| "time_profile/curr_logprobs_and_update": 0.26137867687793914, |
| "time_profile/old_logps": 33.66459304280579, |
| "time_profile/ref_logps": 33.77252321410924, |
| "time_profile/reward": 0.004362174309790134, |
| "time_profile/text_rollout": 486.8325478192419, |
| "train/gen_step": 6020.0, |
| "und/advantages_abs_mean": 0.0693359375, |
| "und/advantages_max": 2.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.625, |
| "und/advantages_std": 0.27044573426246643, |
| "und/clip_ratio/high_mean": 0.0046464357401418965, |
| "und/clip_ratio/low_mean": 0.0008429612207692116, |
| "und/clip_ratio/region_mean": 0.005489396939083235, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.8828125, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.8125, |
| "und/kl": 1.0781144749053055, |
| "und/loss": 0.00424630767565759, |
| "und/prompt/max_length": 247.0, |
| "und/prompt/mean_length": 145.0, |
| "und/prompt/min_length": 43.0, |
| "und/reward": 0.287109375, |
| "und/reward_std": 0.3735242784023285 |
| }, |
| { |
| "epoch": 1.2177742193755003, |
| "grad_norm": 0.008688083849847317, |
| "learning_rate": 1e-05, |
| "loss": 0.0111, |
| "step": 96, |
| "time_profile/curr_logprobs_and_update": 0.2616057716368232, |
| "train/gen_step": 6084.0, |
| "und/clip_ratio/high_mean": 0.00013542917804443277, |
| "und/clip_ratio/low_mean": 4.882812572759576e-05, |
| "und/clip_ratio/region_mean": 0.00018425730377202854, |
| "und/kl": 0.27596294990507886, |
| "und/loss": 0.002798294794956746 |
| }, |
| { |
| "epoch": 1.2305844675740594, |
| "grad_norm": 0.003840341465547681, |
| "learning_rate": 1e-05, |
| "loss": 0.0055, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.0546875, |
| "rewards/strict_format_reward_func/std": 0.15728822350502014, |
| "step": 97, |
| "time_profile/curr_logprobs_and_update": 0.26164151464763563, |
| "time_profile/old_logps": 33.694718721322715, |
| "time_profile/ref_logps": 33.787934163585305, |
| "time_profile/reward": 0.004452788271009922, |
| "time_profile/text_rollout": 518.2917092712596, |
| "train/gen_step": 6148.0, |
| "und/advantages_abs_mean": 0.088623046875, |
| "und/advantages_max": 0.4375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.3125, |
| "und/advantages_std": 0.14899368584156036, |
| "und/clip_ratio/high_mean": 0.000445808833319461, |
| "und/clip_ratio/low_mean": 0.0001805975625757128, |
| "und/clip_ratio/region_mean": 0.0006264063958951738, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.91015625, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.546875, |
| "und/kl": 0.08030579777550884, |
| "und/loss": 0.008490518508324385, |
| "und/prompt/max_length": 313.0, |
| "und/prompt/mean_length": 237.5, |
| "und/prompt/min_length": 162.0, |
| "und/reward": 0.0673828125, |
| "und/reward_std": 0.1709035038948059 |
| }, |
| { |
| "epoch": 1.2433947157726182, |
| "grad_norm": 0.002296620514243841, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "step": 98, |
| "time_profile/curr_logprobs_and_update": 0.2616005388117628, |
| "train/gen_step": 6212.0, |
| "und/clip_ratio/high_mean": 0.0002009032541536726, |
| "und/clip_ratio/low_mean": 5.5803571740398183e-05, |
| "und/clip_ratio/region_mean": 0.0002567068258940708, |
| "und/kl": 0.05221230116148945, |
| "und/loss": 0.0016822464920096536 |
| }, |
| { |
| "epoch": 1.256204963971177, |
| "grad_norm": 0.0028974718879908323, |
| "learning_rate": 1e-05, |
| "loss": 0.0042, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.0546875, |
| "rewards/strict_format_reward_func/std": 0.15728822350502014, |
| "step": 99, |
| "time_profile/curr_logprobs_and_update": 0.26193443170632236, |
| "time_profile/old_logps": 33.68688790965825, |
| "time_profile/ref_logps": 33.75524262525141, |
| "time_profile/reward": 0.004406670108437538, |
| "time_profile/text_rollout": 518.5918508702889, |
| "train/gen_step": 6276.0, |
| "und/advantages_abs_mean": 0.095458984375, |
| "und/advantages_max": 0.4375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.3125, |
| "und/advantages_std": 0.1546332687139511, |
| "und/clip_ratio/high_mean": 0.00018194289805251174, |
| "und/clip_ratio/low_mean": 4.340277882874943e-05, |
| "und/clip_ratio/region_mean": 0.00022534567688126117, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.91796875, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.515625, |
| "und/kl": 0.10376544366590679, |
| "und/loss": 0.0064854817031232415, |
| "und/prompt/max_length": 313.0, |
| "und/prompt/mean_length": 237.5, |
| "und/prompt/min_length": 162.0, |
| "und/reward": 0.0732421875, |
| "und/reward_std": 0.17696848511695862 |
| }, |
| { |
| "epoch": 1.2690152121697358, |
| "grad_norm": 0.002257066546007991, |
| "learning_rate": 1e-05, |
| "loss": 0.0048, |
| "step": 100, |
| "time_profile/curr_logprobs_and_update": 0.26184689076035284, |
| "train/gen_step": 6340.0, |
| "und/clip_ratio/high_mean": 5.425347262644209e-05, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 5.425347262644209e-05, |
| "und/kl": 0.12037181136838626, |
| "und/loss": 0.0016251774967486199 |
| }, |
| { |
| "epoch": 1.2818254603682946, |
| "grad_norm": 0.04131701961159706, |
| "learning_rate": 1e-05, |
| "loss": 0.0128, |
| "rewards/correctness_reward_func/mean": 0.28125, |
| "rewards/correctness_reward_func/std": 0.7007648944854736, |
| "rewards/strict_format_reward_func/mean": 0.3046875, |
| "rewards/strict_format_reward_func/std": 0.24587368965148926, |
| "step": 101, |
| "time_profile/curr_logprobs_and_update": 0.2618017203931231, |
| "time_profile/old_logps": 33.654254213906825, |
| "time_profile/ref_logps": 33.73097015917301, |
| "time_profile/reward": 0.004532042890787125, |
| "time_profile/text_rollout": 455.13132797554135, |
| "train/gen_step": 6404.0, |
| "und/advantages_abs_mean": 0.5234375, |
| "und/advantages_max": 1.875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.5, |
| "und/advantages_std": 0.676927924156189, |
| "und/clip_ratio/high_mean": 0.00020775113807758316, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.00020775113807758316, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.73046875, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.015625, |
| "und/kl": 0.11258987631299533, |
| "und/loss": 0.01647166971235947, |
| "und/prompt/max_length": 56.0, |
| "und/prompt/mean_length": 52.5, |
| "und/prompt/min_length": 49.0, |
| "und/reward": 0.7099609375, |
| "und/reward_std": 0.905170202255249 |
| }, |
| { |
| "epoch": 1.2946357085668536, |
| "grad_norm": 0.05473562330007553, |
| "learning_rate": 1e-05, |
| "loss": 0.0355, |
| "step": 102, |
| "time_profile/curr_logprobs_and_update": 0.261388782368158, |
| "train/gen_step": 6468.0, |
| "und/clip_ratio/high_mean": 0.0005066352496214677, |
| "und/clip_ratio/low_mean": 9.820867489906959e-05, |
| "und/clip_ratio/region_mean": 0.0006048439317964949, |
| "und/kl": 0.8876906962541398, |
| "und/loss": 0.001254691684152931 |
| }, |
| { |
| "epoch": 1.3074459567654122, |
| "grad_norm": 0.034072067588567734, |
| "learning_rate": 1e-05, |
| "loss": -0.0061, |
| "rewards/correctness_reward_func/mean": 0.0625, |
| "rewards/correctness_reward_func/std": 0.35073620080947876, |
| "rewards/strict_format_reward_func/mean": 0.2265625, |
| "rewards/strict_format_reward_func/std": 0.250866562128067, |
| "step": 103, |
| "time_profile/curr_logprobs_and_update": 0.26150604666327126, |
| "time_profile/old_logps": 33.661563923582435, |
| "time_profile/ref_logps": 33.73078432120383, |
| "time_profile/reward": 0.004420598968863487, |
| "time_profile/text_rollout": 499.30165772140026, |
| "train/gen_step": 6532.0, |
| "und/advantages_abs_mean": 0.116455078125, |
| "und/advantages_max": 1.9375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.9375, |
| "und/advantages_std": 0.3238520324230194, |
| "und/clip_ratio/high_mean": 0.0007794650773575995, |
| "und/clip_ratio/low_mean": 0.0006103413725213613, |
| "und/clip_ratio/region_mean": 0.001389806446240982, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.703125, |
| "und/completion/min_length": 190.0, |
| "und/frac_reward_zero_std": 0.65625, |
| "und/kl": 0.0528246548055904, |
| "und/loss": -0.01438412336096917, |
| "und/prompt/max_length": 313.0, |
| "und/prompt/mean_length": 178.0, |
| "und/prompt/min_length": 43.0, |
| "und/reward": 0.2900390625, |
| "und/reward_std": 0.45152318477630615 |
| }, |
| { |
| "epoch": 1.3202562049639712, |
| "grad_norm": 0.005107598844915628, |
| "learning_rate": 1e-05, |
| "loss": 0.0064, |
| "step": 104, |
| "time_profile/curr_logprobs_and_update": 0.26163111737696454, |
| "train/gen_step": 6596.0, |
| "und/clip_ratio/high_mean": 5.918560782447457e-05, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 5.918560782447457e-05, |
| "und/kl": 0.15930140123236924, |
| "und/loss": 0.0028715015233444774 |
| }, |
| { |
| "epoch": 1.33306645316253, |
| "grad_norm": 0.028493041172623634, |
| "learning_rate": 1e-05, |
| "loss": 0.0042, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.1953125, |
| "rewards/strict_format_reward_func/std": 0.24587368965148926, |
| "step": 105, |
| "time_profile/curr_logprobs_and_update": 0.2615614668466151, |
| "time_profile/old_logps": 33.699232946150005, |
| "time_profile/ref_logps": 33.78243537712842, |
| "time_profile/reward": 0.004495236091315746, |
| "time_profile/text_rollout": 491.3853238923475, |
| "train/gen_step": 6660.0, |
| "und/advantages_abs_mean": 0.205078125, |
| "und/advantages_max": 0.4375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.3125, |
| "und/advantages_std": 0.226649209856987, |
| "und/clip_ratio/high_mean": 0.00030362879260792397, |
| "und/clip_ratio/low_mean": 9.527438669465482e-05, |
| "und/clip_ratio/region_mean": 0.0003989031793025788, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.7734375, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.015625, |
| "und/kl": 0.10507874877657741, |
| "und/loss": -0.0043832145364604, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 162.0, |
| "und/prompt/min_length": 162.0, |
| "und/reward": 0.1640625, |
| "und/reward_std": 0.23499490320682526 |
| }, |
| { |
| "epoch": 1.3458767013610888, |
| "grad_norm": 0.009856931865215302, |
| "learning_rate": 1e-05, |
| "loss": 0.0113, |
| "step": 106, |
| "time_profile/curr_logprobs_and_update": 0.2615844376850873, |
| "train/gen_step": 6724.0, |
| "und/clip_ratio/high_mean": 0.00010460711928317323, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.00010460711928317323, |
| "und/kl": 0.2839002070031711, |
| "und/loss": 0.0006757988594472408 |
| }, |
| { |
| "epoch": 1.3586869495596476, |
| "grad_norm": 0.013232600875198841, |
| "learning_rate": 1e-05, |
| "loss": 0.0067, |
| "rewards/correctness_reward_func/mean": 0.03125, |
| "rewards/correctness_reward_func/std": 0.25, |
| "rewards/strict_format_reward_func/mean": 0.2578125, |
| "rewards/strict_format_reward_func/std": 0.25185325741767883, |
| "step": 107, |
| "time_profile/curr_logprobs_and_update": 0.26170188521791715, |
| "time_profile/old_logps": 33.66488418634981, |
| "time_profile/ref_logps": 33.74126262869686, |
| "time_profile/reward": 0.004568587988615036, |
| "time_profile/text_rollout": 455.8338515488431, |
| "train/gen_step": 6788.0, |
| "und/advantages_abs_mean": 0.17236328125, |
| "und/advantages_max": 2.0, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.5, |
| "und/advantages_std": 0.2258382886648178, |
| "und/clip_ratio/high_mean": 0.0003017742856172845, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0003017742856172845, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.767578125, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.140625, |
| "und/kl": 0.20220415030053118, |
| "und/loss": 0.008677718930357514, |
| "und/prompt/max_length": 58.0, |
| "und/prompt/mean_length": 55.5, |
| "und/prompt/min_length": 53.0, |
| "und/reward": 0.302734375, |
| "und/reward_std": 0.26385733485221863 |
| }, |
| { |
| "epoch": 1.3714971977582064, |
| "grad_norm": 0.01322892215102911, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "step": 108, |
| "time_profile/curr_logprobs_and_update": 0.2693607303954195, |
| "train/gen_step": 6852.0, |
| "und/clip_ratio/high_mean": 0.0003047961799893528, |
| "und/clip_ratio/low_mean": 0.0001023065487970598, |
| "und/clip_ratio/region_mean": 0.0004071027287864126, |
| "und/kl": 0.04968085115979193, |
| "und/loss": 0.0027706179830602196 |
| }, |
| { |
| "epoch": 1.3843074459567655, |
| "grad_norm": 0.012319354340434074, |
| "learning_rate": 1e-05, |
| "loss": 0.0078, |
| "rewards/correctness_reward_func/mean": 0.375, |
| "rewards/correctness_reward_func/std": 0.7867957949638367, |
| "rewards/strict_format_reward_func/mean": 0.375, |
| "rewards/strict_format_reward_func/std": 0.2182178944349289, |
| "step": 109, |
| "time_profile/curr_logprobs_and_update": 0.2616202123608673, |
| "time_profile/old_logps": 33.62943078484386, |
| "time_profile/ref_logps": 33.73900862503797, |
| "time_profile/reward": 0.004435631446540356, |
| "time_profile/text_rollout": 454.5818900093436, |
| "train/gen_step": 6916.0, |
| "und/advantages_abs_mean": 0.563232421875, |
| "und/advantages_max": 1.875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.6875, |
| "und/advantages_std": 0.72254878282547, |
| "und/clip_ratio/high_mean": 0.0008765930215304252, |
| "und/clip_ratio/low_mean": 9.084302291739732e-05, |
| "und/clip_ratio/region_mean": 0.0009674360444478225, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.5234375, |
| "und/completion/min_length": 180.0, |
| "und/frac_reward_zero_std": 0.015625, |
| "und/kl": 0.10922401638526935, |
| "und/loss": 0.014170625921906321, |
| "und/prompt/max_length": 51.0, |
| "und/prompt/mean_length": 48.5, |
| "und/prompt/min_length": 46.0, |
| "und/reward": 0.7587890625, |
| "und/reward_std": 0.916750431060791 |
| }, |
| { |
| "epoch": 1.3971176941553243, |
| "grad_norm": 0.014209937304258347, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "step": 110, |
| "time_profile/curr_logprobs_and_update": 0.2613295341434423, |
| "train/gen_step": 6980.0, |
| "und/clip_ratio/high_mean": 0.00018801813712343574, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.00018801813712343574, |
| "und/kl": 0.06319543437712127, |
| "und/loss": 0.0012067200150340796 |
| }, |
| { |
| "epoch": 1.409927942353883, |
| "grad_norm": 0.00816685613244772, |
| "learning_rate": 1e-05, |
| "loss": 0.0138, |
| "rewards/correctness_reward_func/mean": 0.9375, |
| "rewards/correctness_reward_func/std": 1.0059348344802856, |
| "rewards/strict_format_reward_func/mean": 0.25, |
| "rewards/strict_format_reward_func/std": 0.2519763112068176, |
| "step": 111, |
| "time_profile/curr_logprobs_and_update": 0.26151229023525957, |
| "time_profile/old_logps": 33.65623056329787, |
| "time_profile/ref_logps": 33.743903799913824, |
| "time_profile/reward": 0.004555624909698963, |
| "time_profile/text_rollout": 499.53031921666116, |
| "train/gen_step": 7044.0, |
| "und/advantages_abs_mean": 0.22607421875, |
| "und/advantages_max": 2.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -2.1875, |
| "und/advantages_std": 0.4984687864780426, |
| "und/clip_ratio/high_mean": 0.00038183126162039116, |
| "und/clip_ratio/low_mean": 0.00011579119382076897, |
| "und/clip_ratio/region_mean": 0.0004976224590791389, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.775390625, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.640625, |
| "und/kl": 0.3358783017611131, |
| "und/loss": 0.009991740759346612, |
| "und/prompt/max_length": 313.0, |
| "und/prompt/mean_length": 178.5, |
| "und/prompt/min_length": 44.0, |
| "und/reward": 1.1123046875, |
| "und/reward_std": 1.224579095840454 |
| }, |
| { |
| "epoch": 1.4227381905524419, |
| "grad_norm": 0.0051344470120966434, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "step": 112, |
| "time_profile/curr_logprobs_and_update": 0.2616101046296535, |
| "train/gen_step": 7108.0, |
| "und/clip_ratio/high_mean": 5.139802669873461e-05, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 5.139802669873461e-05, |
| "und/kl": 0.05211045581381768, |
| "und/loss": 0.0019724031756140903 |
| }, |
| { |
| "epoch": 1.4355484387510007, |
| "grad_norm": 0.05978088825941086, |
| "learning_rate": 1e-05, |
| "loss": 0.0074, |
| "rewards/correctness_reward_func/mean": 1.4375, |
| "rewards/correctness_reward_func/std": 0.9063270092010498, |
| "rewards/strict_format_reward_func/mean": 0.4921875, |
| "rewards/strict_format_reward_func/std": 0.0625, |
| "step": 113, |
| "time_profile/curr_logprobs_and_update": 0.2615887124411529, |
| "time_profile/old_logps": 33.653056289069355, |
| "time_profile/ref_logps": 33.754895557649434, |
| "time_profile/reward": 0.0044929469004273415, |
| "time_profile/text_rollout": 454.0134041188285, |
| "train/gen_step": 7172.0, |
| "und/advantages_abs_mean": 0.576416015625, |
| "und/advantages_max": 1.8125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -2.1875, |
| "und/advantages_std": 0.769360363483429, |
| "und/clip_ratio/high_mean": 5.008012885809876e-05, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 5.008012885809876e-05, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 253.978515625, |
| "und/completion/min_length": 151.0, |
| "und/frac_reward_zero_std": 0.234375, |
| "und/kl": 0.07507353127584793, |
| "und/loss": 0.0015328295562540006, |
| "und/prompt/max_length": 49.0, |
| "und/prompt/mean_length": 46.5, |
| "und/prompt/min_length": 44.0, |
| "und/reward": 1.9208984375, |
| "und/reward_std": 0.9253405928611755 |
| }, |
| { |
| "epoch": 1.4483586869495597, |
| "grad_norm": 0.09742525219917297, |
| "learning_rate": 1e-05, |
| "loss": 0.0053, |
| "step": 114, |
| "time_profile/curr_logprobs_and_update": 0.2612951010960387, |
| "train/gen_step": 7236.0, |
| "und/clip_ratio/high_mean": 0.00040703625199967064, |
| "und/clip_ratio/low_mean": 0.0002526224052417092, |
| "und/clip_ratio/region_mean": 0.0006596586572413798, |
| "und/kl": 0.13804292841814458, |
| "und/loss": 0.013488775107816764 |
| }, |
| { |
| "epoch": 1.4611689351481185, |
| "grad_norm": 0.04351665452122688, |
| "learning_rate": 1e-05, |
| "loss": -0.002, |
| "rewards/correctness_reward_func/mean": 0.0625, |
| "rewards/correctness_reward_func/std": 0.35073620080947876, |
| "rewards/strict_format_reward_func/mean": 0.3359375, |
| "rewards/strict_format_reward_func/std": 0.2366211861371994, |
| "step": 115, |
| "time_profile/curr_logprobs_and_update": 0.26138295279815793, |
| "time_profile/old_logps": 33.6624674256891, |
| "time_profile/ref_logps": 33.752434914931655, |
| "time_profile/reward": 0.004297954961657524, |
| "time_profile/text_rollout": 454.55648574698716, |
| "train/gen_step": 7300.0, |
| "und/advantages_abs_mean": 0.22314453125, |
| "und/advantages_max": 1.8125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.9375, |
| "und/advantages_std": 0.378530353307724, |
| "und/clip_ratio/high_mean": 0.00232754351964104, |
| "und/clip_ratio/low_mean": 0.00021809619283885695, |
| "und/clip_ratio/region_mean": 0.0025456397052039392, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.7421875, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.21875, |
| "und/kl": 0.07022856344701722, |
| "und/loss": -0.011109988249302205, |
| "und/prompt/max_length": 52.0, |
| "und/prompt/mean_length": 48.5, |
| "und/prompt/min_length": 45.0, |
| "und/reward": 0.4130859375, |
| "und/reward_std": 0.4497930109500885 |
| }, |
| { |
| "epoch": 1.4739791833466773, |
| "grad_norm": 0.041889388114213943, |
| "learning_rate": 1e-05, |
| "loss": 0.0074, |
| "step": 116, |
| "time_profile/curr_logprobs_and_update": 0.26131885705399327, |
| "train/gen_step": 7364.0, |
| "und/clip_ratio/high_mean": 0.00015886059190961532, |
| "und/clip_ratio/low_mean": 9.917301213135943e-05, |
| "und/clip_ratio/region_mean": 0.00025803360404097475, |
| "und/kl": 0.1826191512591322, |
| "und/loss": 0.003956302835831593 |
| }, |
| { |
| "epoch": 1.4867894315452361, |
| "grad_norm": 0.03012249618768692, |
| "learning_rate": 1e-05, |
| "loss": 0.0046, |
| "rewards/correctness_reward_func/mean": 0.03125, |
| "rewards/correctness_reward_func/std": 0.25, |
| "rewards/strict_format_reward_func/mean": 0.265625, |
| "rewards/strict_format_reward_func/std": 0.2514837086200714, |
| "step": 117, |
| "time_profile/curr_logprobs_and_update": 0.2616950735828141, |
| "time_profile/old_logps": 33.65449043828994, |
| "time_profile/ref_logps": 33.74488543625921, |
| "time_profile/reward": 0.0031747594475746155, |
| "time_profile/text_rollout": 486.8246115138754, |
| "train/gen_step": 7428.0, |
| "und/advantages_abs_mean": 0.090576171875, |
| "und/advantages_max": 2.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.4375, |
| "und/advantages_std": 0.304784893989563, |
| "und/clip_ratio/high_mean": 0.0004131392852286808, |
| "und/clip_ratio/low_mean": 0.00020289487292757258, |
| "und/clip_ratio/region_mean": 0.0006160341581562534, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.845703125, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.703125, |
| "und/kl": 0.14989926843554713, |
| "und/loss": 0.0282161407434387, |
| "und/prompt/max_length": 247.0, |
| "und/prompt/mean_length": 145.0, |
| "und/prompt/min_length": 43.0, |
| "und/reward": 0.2861328125, |
| "und/reward_std": 0.37623000144958496 |
| }, |
| { |
| "epoch": 1.499599679743795, |
| "grad_norm": 0.006430523935705423, |
| "learning_rate": 1e-05, |
| "loss": 0.0048, |
| "step": 118, |
| "time_profile/curr_logprobs_and_update": 0.26145954702224117, |
| "train/gen_step": 7492.0, |
| "und/clip_ratio/high_mean": 0.0002027803311648313, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0002027803311648313, |
| "und/kl": 0.12073310048435815, |
| "und/loss": 0.002307371724555196 |
| }, |
| { |
| "epoch": 1.512409927942354, |
| "grad_norm": 0.00652286084368825, |
| "learning_rate": 1e-05, |
| "loss": 0.0079, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.1328125, |
| "rewards/strict_format_reward_func/std": 0.22257846593856812, |
| "step": 119, |
| "time_profile/curr_logprobs_and_update": 0.2616856778477086, |
| "time_profile/old_logps": 33.690122249536216, |
| "time_profile/ref_logps": 33.760620589368045, |
| "time_profile/reward": 0.004584756679832935, |
| "time_profile/text_rollout": 491.3738481281325, |
| "train/gen_step": 7556.0, |
| "und/advantages_abs_mean": 0.17724609375, |
| "und/advantages_max": 0.4375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.375, |
| "und/advantages_std": 0.210708886384964, |
| "und/clip_ratio/high_mean": 0.0003724069974850863, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0003724069974850863, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.8046875, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.0625, |
| "und/kl": 0.11169646086636931, |
| "und/loss": 0.006852565979897918, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 162.0, |
| "und/prompt/min_length": 162.0, |
| "und/reward": 0.14453125, |
| "und/reward_std": 0.22688518464565277 |
| }, |
| { |
| "epoch": 1.5252201761409128, |
| "grad_norm": 0.010090266354382038, |
| "learning_rate": 1e-05, |
| "loss": 0.0047, |
| "step": 120, |
| "time_profile/curr_logprobs_and_update": 0.2616492405504687, |
| "train/gen_step": 7620.0, |
| "und/clip_ratio/high_mean": 0.00010801373355207033, |
| "und/clip_ratio/low_mean": 8.448401058558375e-05, |
| "und/clip_ratio/region_mean": 0.00019249774413765408, |
| "und/kl": 0.11743137614394072, |
| "und/loss": 0.004808933234471624 |
| }, |
| { |
| "epoch": 1.5380304243394716, |
| "grad_norm": 0.010744316503405571, |
| "learning_rate": 1e-05, |
| "loss": -0.0028, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.1328125, |
| "rewards/strict_format_reward_func/std": 0.22257846593856812, |
| "step": 121, |
| "time_profile/curr_logprobs_and_update": 0.2618774699512869, |
| "time_profile/old_logps": 33.68734922725707, |
| "time_profile/ref_logps": 33.784588277339935, |
| "time_profile/reward": 0.004522095434367657, |
| "time_profile/text_rollout": 491.46461437735707, |
| "train/gen_step": 7684.0, |
| "und/advantages_abs_mean": 0.173095703125, |
| "und/advantages_max": 0.4375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.3125, |
| "und/advantages_std": 0.20822730660438538, |
| "und/clip_ratio/high_mean": 0.00026392990548629314, |
| "und/clip_ratio/low_mean": 0.00010926573668257333, |
| "und/clip_ratio/region_mean": 0.00037319564216886647, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.8203125, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.078125, |
| "und/kl": 0.09060025405779015, |
| "und/loss": -0.003209088767432533, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 162.0, |
| "und/prompt/min_length": 162.0, |
| "und/reward": 0.1337890625, |
| "und/reward_std": 0.2215646654367447 |
| }, |
| { |
| "epoch": 1.5508406725380304, |
| "grad_norm": 0.9860634803771973, |
| "learning_rate": 1e-05, |
| "loss": 0.0568, |
| "step": 122, |
| "time_profile/curr_logprobs_and_update": 0.2615589259658009, |
| "train/gen_step": 7748.0, |
| "und/clip_ratio/high_mean": 0.0002418396543362178, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0002418396543362178, |
| "und/kl": 1.4184919325416558, |
| "und/loss": 0.0028312649810686707 |
| }, |
| { |
| "epoch": 1.5636509207365892, |
| "grad_norm": 0.031551264226436615, |
| "learning_rate": 1e-05, |
| "loss": 2.86, |
| "rewards/correctness_reward_func/mean": 0.75, |
| "rewards/correctness_reward_func/std": 0.9759001135826111, |
| "rewards/strict_format_reward_func/mean": 0.390625, |
| "rewards/strict_format_reward_func/std": 0.2083333432674408, |
| "step": 123, |
| "time_profile/curr_logprobs_and_update": 0.261691846215399, |
| "time_profile/old_logps": 33.64521975629032, |
| "time_profile/ref_logps": 33.757645015604794, |
| "time_profile/reward": 0.004097883589565754, |
| "time_profile/text_rollout": 453.70345996133983, |
| "train/gen_step": 7812.0, |
| "und/advantages_abs_mean": 0.382568359375, |
| "und/advantages_max": 1.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -2.1875, |
| "und/advantages_std": 0.5867711305618286, |
| "und/clip_ratio/high_mean": 0.0006388125475496054, |
| "und/clip_ratio/low_mean": 0.0006393747644324321, |
| "und/clip_ratio/region_mean": 0.0012781873083440587, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.76953125, |
| "und/completion/min_length": 239.0, |
| "und/frac_reward_zero_std": 0.125, |
| "und/kl": 0.0320791233170894, |
| "und/loss": 0.0024011894063278305, |
| "und/prompt/max_length": 46.0, |
| "und/prompt/mean_length": 44.5, |
| "und/prompt/min_length": 43.0, |
| "und/reward": 1.21875, |
| "und/reward_std": 1.0983887910842896 |
| }, |
| { |
| "epoch": 1.5764611689351482, |
| "grad_norm": 0.03529511019587517, |
| "learning_rate": 1e-05, |
| "loss": 0.0066, |
| "step": 124, |
| "time_profile/curr_logprobs_and_update": 0.26139320741640404, |
| "train/gen_step": 7876.0, |
| "und/clip_ratio/high_mean": 0.0008959227816376369, |
| "und/clip_ratio/low_mean": 0.00013536133337765932, |
| "und/clip_ratio/region_mean": 0.0010312841150152963, |
| "und/kl": 0.16470841343107168, |
| "und/loss": 0.004291785090572375 |
| }, |
| { |
| "epoch": 1.589271417133707, |
| "grad_norm": 0.009518980048596859, |
| "learning_rate": 1e-05, |
| "loss": 0.0121, |
| "rewards/correctness_reward_func/mean": 0.03125, |
| "rewards/correctness_reward_func/std": 0.25, |
| "rewards/strict_format_reward_func/mean": 0.0859375, |
| "rewards/strict_format_reward_func/std": 0.19012710452079773, |
| "step": 125, |
| "time_profile/curr_logprobs_and_update": 0.26169546390883625, |
| "time_profile/old_logps": 33.664723821915686, |
| "time_profile/ref_logps": 33.749822195619345, |
| "time_profile/reward": 0.004408887587487698, |
| "time_profile/text_rollout": 505.58012927044183, |
| "train/gen_step": 7940.0, |
| "und/advantages_abs_mean": 0.13330078125, |
| "und/advantages_max": 2.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.5625, |
| "und/advantages_std": 0.26542437076568604, |
| "und/clip_ratio/high_mean": 0.001534490605990868, |
| "und/clip_ratio/low_mean": 0.0003314741697977297, |
| "und/clip_ratio/region_mean": 0.0018659647794265766, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.87109375, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.4375, |
| "und/kl": 0.08085775800282136, |
| "und/loss": 0.009419288670414971, |
| "und/prompt/max_length": 247.0, |
| "und/prompt/mean_length": 204.5, |
| "und/prompt/min_length": 162.0, |
| "und/reward": 0.103515625, |
| "und/reward_std": 0.29348325729370117 |
| }, |
| { |
| "epoch": 1.6020816653322658, |
| "grad_norm": 0.010913106612861156, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "step": 126, |
| "time_profile/curr_logprobs_and_update": 0.2617497799510602, |
| "train/gen_step": 8004.0, |
| "und/clip_ratio/high_mean": 0.00018722502863965929, |
| "und/clip_ratio/low_mean": 0.00034632114693522453, |
| "und/clip_ratio/region_mean": 0.0005335461755748838, |
| "und/kl": 0.06019564852613257, |
| "und/loss": 0.002116124094982297 |
| }, |
| { |
| "epoch": 1.6148919135308246, |
| "grad_norm": 0.06803760677576065, |
| "learning_rate": 1e-05, |
| "loss": 0.0009, |
| "rewards/correctness_reward_func/mean": 0.96875, |
| "rewards/correctness_reward_func/std": 1.0074130296707153, |
| "rewards/strict_format_reward_func/mean": 0.4375, |
| "rewards/strict_format_reward_func/std": 0.1666666716337204, |
| "step": 127, |
| "time_profile/curr_logprobs_and_update": 0.2617532514850609, |
| "time_profile/old_logps": 33.6751423580572, |
| "time_profile/ref_logps": 33.763203986920416, |
| "time_profile/reward": 0.004775381647050381, |
| "time_profile/text_rollout": 471.52733379229903, |
| "train/gen_step": 8068.0, |
| "und/advantages_abs_mean": 0.903564453125, |
| "und/advantages_max": 2.0625, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.9375, |
| "und/advantages_std": 1.0012528896331787, |
| "und/clip_ratio/high_mean": 0.00015197079119388945, |
| "und/clip_ratio/low_mean": 0.0012435419848770835, |
| "und/clip_ratio/region_mean": 0.001395512772432994, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.599609375, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.015625, |
| "und/kl": 0.0706367947277613, |
| "und/loss": -0.004468285508210101, |
| "und/prompt/max_length": 156.0, |
| "und/prompt/mean_length": 102.5, |
| "und/prompt/min_length": 49.0, |
| "und/reward": 1.2705078125, |
| "und/reward_std": 1.0752369165420532 |
| }, |
| { |
| "epoch": 1.6277021617293834, |
| "grad_norm": 0.07240945845842361, |
| "learning_rate": 1e-05, |
| "loss": 30.4224, |
| "step": 128, |
| "time_profile/curr_logprobs_and_update": 0.2616008282493567, |
| "train/gen_step": 8132.0, |
| "und/clip_ratio/high_mean": 0.0006445251565310173, |
| "und/clip_ratio/low_mean": 0.00013259582192404196, |
| "und/clip_ratio/region_mean": 0.0007771209748170804, |
| "und/kl": 760.5635505191167, |
| "und/loss": 243.34435985563323 |
| }, |
| { |
| "epoch": 1.6405124099279424, |
| "grad_norm": 0.06157054379582405, |
| "learning_rate": 1e-05, |
| "loss": 0.0101, |
| "rewards/correctness_reward_func/mean": 0.15625, |
| "rewards/correctness_reward_func/std": 0.5409794449806213, |
| "rewards/strict_format_reward_func/mean": 0.2421875, |
| "rewards/strict_format_reward_func/std": 0.25185325741767883, |
| "step": 129, |
| "time_profile/curr_logprobs_and_update": 0.26170297838689294, |
| "time_profile/old_logps": 33.68404742050916, |
| "time_profile/ref_logps": 33.77305310498923, |
| "time_profile/reward": 0.004530956037342548, |
| "time_profile/text_rollout": 473.9377162186429, |
| "train/gen_step": 8196.0, |
| "und/advantages_abs_mean": 0.3798828125, |
| "und/advantages_max": 2.125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.0625, |
| "und/advantages_std": 0.5802733898162842, |
| "und/clip_ratio/high_mean": 0.0007628962412127294, |
| "und/clip_ratio/low_mean": 0.0006278082109929528, |
| "und/clip_ratio/region_mean": 0.0013907044594816398, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.736328125, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.015625, |
| "und/kl": 0.1402410990849603, |
| "und/loss": 0.008292756974697113, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 110.5, |
| "und/prompt/min_length": 59.0, |
| "und/reward": 0.353515625, |
| "und/reward_std": 0.6494219303131104 |
| }, |
| { |
| "epoch": 1.6533226581265013, |
| "grad_norm": 0.03736671432852745, |
| "learning_rate": 1e-05, |
| "loss": 1.3177, |
| "step": 130, |
| "time_profile/curr_logprobs_and_update": 0.26144938450306654, |
| "train/gen_step": 8260.0, |
| "und/clip_ratio/high_mean": 0.000294210301944986, |
| "und/clip_ratio/low_mean": 0.001453026579838479, |
| "und/clip_ratio/region_mean": 0.001747236881783465, |
| "und/kl": 32.937790766736725, |
| "und/loss": 0.008295642333791875 |
| }, |
| { |
| "epoch": 1.66613290632506, |
| "grad_norm": 0.04435932636260986, |
| "learning_rate": 1e-05, |
| "loss": 15.2847, |
| "rewards/correctness_reward_func/mean": 0.6875, |
| "rewards/correctness_reward_func/std": 0.9574271440505981, |
| "rewards/strict_format_reward_func/mean": 0.3203125, |
| "rewards/strict_format_reward_func/std": 0.24180518090724945, |
| "step": 131, |
| "time_profile/curr_logprobs_and_update": 0.26175846286059823, |
| "time_profile/old_logps": 33.65911100897938, |
| "time_profile/ref_logps": 33.72500569187105, |
| "time_profile/reward": 0.004387730732560158, |
| "time_profile/text_rollout": 455.486419968307, |
| "train/gen_step": 8324.0, |
| "und/advantages_abs_mean": 0.45947265625, |
| "und/advantages_max": 1.25, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -2.1875, |
| "und/advantages_std": 0.6482231616973877, |
| "und/clip_ratio/high_mean": 0.0007765535956423264, |
| "und/clip_ratio/low_mean": 0.0037881125645071734, |
| "und/clip_ratio/region_mean": 0.004564666138321627, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.681640625, |
| "und/completion/min_length": 252.0, |
| "und/frac_reward_zero_std": 0.078125, |
| "und/kl": 382.2992160025169, |
| "und/loss": 2.063884374489703, |
| "und/prompt/max_length": 55.0, |
| "und/prompt/mean_length": 54.0, |
| "und/prompt/min_length": 53.0, |
| "und/reward": 1.0341796875, |
| "und/reward_std": 1.1236227750778198 |
| }, |
| { |
| "epoch": 1.6789431545236189, |
| "grad_norm": 0.055891405791044235, |
| "learning_rate": 1e-05, |
| "loss": 0.0282, |
| "step": 132, |
| "time_profile/curr_logprobs_and_update": 0.2613213101722067, |
| "train/gen_step": 8388.0, |
| "und/clip_ratio/high_mean": 0.00035001601281692274, |
| "und/clip_ratio/low_mean": 8.311169949593022e-05, |
| "und/clip_ratio/region_mean": 0.0004331277195888106, |
| "und/kl": 0.7055800975940656, |
| "und/loss": 0.05676676309667528 |
| }, |
| { |
| "epoch": 1.6917534027221777, |
| "grad_norm": 0.04796978086233139, |
| "learning_rate": 1e-05, |
| "loss": 0.1329, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.1640625, |
| "rewards/strict_format_reward_func/std": 0.2366211861371994, |
| "step": 133, |
| "time_profile/curr_logprobs_and_update": 0.2614839215821121, |
| "time_profile/old_logps": 33.681789374910295, |
| "time_profile/ref_logps": 33.78480255138129, |
| "time_profile/reward": 0.004319341853260994, |
| "time_profile/text_rollout": 491.34117817133665, |
| "train/gen_step": 8452.0, |
| "und/advantages_abs_mean": 0.18505859375, |
| "und/advantages_max": 2.0, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.5, |
| "und/advantages_std": 0.2327723652124405, |
| "und/clip_ratio/high_mean": 0.001052155526849674, |
| "und/clip_ratio/low_mean": 0.0005224456072028261, |
| "und/clip_ratio/region_mean": 0.0015746011340525001, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.810546875, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.078125, |
| "und/kl": 3.207622427144088, |
| "und/loss": 0.06492377794347703, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 162.0, |
| "und/prompt/min_length": 162.0, |
| "und/reward": 0.1484375, |
| "und/reward_std": 0.2491423636674881 |
| }, |
| { |
| "epoch": 1.7045636509207367, |
| "grad_norm": 0.0103101572021842, |
| "learning_rate": 1e-05, |
| "loss": 619821.5625, |
| "step": 134, |
| "time_profile/curr_logprobs_and_update": 0.26158440396829974, |
| "train/gen_step": 8516.0, |
| "und/clip_ratio/high_mean": 0.0003344619144627359, |
| "und/clip_ratio/low_mean": 0.003717986841365928, |
| "und/clip_ratio/region_mean": 0.004052448763104621, |
| "und/kl": 15495538.581971796, |
| "und/loss": 442.40379129559733 |
| }, |
| { |
| "epoch": 1.7173738991192953, |
| "grad_norm": 0.03445025533437729, |
| "learning_rate": 1e-05, |
| "loss": 37837.5664, |
| "rewards/correctness_reward_func/mean": 0.15625, |
| "rewards/correctness_reward_func/std": 0.5409794449806213, |
| "rewards/strict_format_reward_func/mean": 0.2421875, |
| "rewards/strict_format_reward_func/std": 0.25185325741767883, |
| "step": 135, |
| "time_profile/curr_logprobs_and_update": 0.26179739559302106, |
| "time_profile/old_logps": 33.6750593688339, |
| "time_profile/ref_logps": 33.769202906638384, |
| "time_profile/reward": 0.00483175553381443, |
| "time_profile/text_rollout": 516.9765401165932, |
| "train/gen_step": 8580.0, |
| "und/advantages_abs_mean": 0.340087890625, |
| "und/advantages_max": 1.8125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.1875, |
| "und/advantages_std": 0.5872920155525208, |
| "und/clip_ratio/high_mean": 0.0005857992764504161, |
| "und/clip_ratio/low_mean": 0.004737480834592134, |
| "und/clip_ratio/region_mean": 0.00532328011104255, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.693359375, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.5, |
| "und/kl": 945939.298322209, |
| "und/loss": 1624.2115884204977, |
| "und/prompt/max_length": 313.0, |
| "und/prompt/mean_length": 234.5, |
| "und/prompt/min_length": 156.0, |
| "und/reward": 0.474609375, |
| "und/reward_std": 0.7803480625152588 |
| }, |
| { |
| "epoch": 1.7301841473178543, |
| "grad_norm": 0.041320931166410446, |
| "learning_rate": 1e-05, |
| "loss": 0.0137, |
| "step": 136, |
| "time_profile/curr_logprobs_and_update": 0.2617494677833747, |
| "train/gen_step": 8644.0, |
| "und/clip_ratio/high_mean": 0.00022294744121609256, |
| "und/clip_ratio/low_mean": 0.00022158692809171043, |
| "und/clip_ratio/region_mean": 0.000444534369307803, |
| "und/kl": 0.3452570588560775, |
| "und/loss": 0.010216924388146253 |
| }, |
| { |
| "epoch": 1.742994395516413, |
| "grad_norm": 0.028058085590600967, |
| "learning_rate": 1e-05, |
| "loss": 0.0285, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.0546875, |
| "rewards/strict_format_reward_func/std": 0.15728822350502014, |
| "step": 137, |
| "time_profile/curr_logprobs_and_update": 0.2618869323923718, |
| "time_profile/old_logps": 33.705197578296065, |
| "time_profile/ref_logps": 33.796635060571134, |
| "time_profile/reward": 0.00447302870452404, |
| "time_profile/text_rollout": 519.2273662136868, |
| "train/gen_step": 8708.0, |
| "und/advantages_abs_mean": 0.087646484375, |
| "und/advantages_max": 0.4375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.25, |
| "und/advantages_std": 0.14817051589488983, |
| "und/clip_ratio/high_mean": 0.0002926721135736443, |
| "und/clip_ratio/low_mean": 0.00014689966701553203, |
| "und/clip_ratio/region_mean": 0.00043957178058917634, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.908203125, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.546875, |
| "und/kl": 0.6957836126966868, |
| "und/loss": 0.15320786734139347, |
| "und/prompt/max_length": 313.0, |
| "und/prompt/mean_length": 237.5, |
| "und/prompt/min_length": 162.0, |
| "und/reward": 0.0673828125, |
| "und/reward_std": 0.1709035038948059 |
| }, |
| { |
| "epoch": 1.755804643714972, |
| "grad_norm": 0.01070038229227066, |
| "learning_rate": 1e-05, |
| "loss": 73915.4844, |
| "step": 138, |
| "time_profile/curr_logprobs_and_update": 0.26177451500552706, |
| "train/gen_step": 8772.0, |
| "und/clip_ratio/high_mean": 0.00010704063606681302, |
| "und/clip_ratio/low_mean": 0.0015135373505472671, |
| "und/clip_ratio/region_mean": 0.0016205779829761013, |
| "und/kl": 1847887.2981159356, |
| "und/loss": 0.01662214804730411 |
| }, |
| { |
| "epoch": 1.768614891913531, |
| "grad_norm": 0.05411059781908989, |
| "learning_rate": 1e-05, |
| "loss": 179351.5938, |
| "rewards/correctness_reward_func/mean": 0.90625, |
| "rewards/correctness_reward_func/std": 1.003466248512268, |
| "rewards/strict_format_reward_func/mean": 0.28125, |
| "rewards/strict_format_reward_func/std": 0.25, |
| "step": 139, |
| "time_profile/curr_logprobs_and_update": 0.26185339997755364, |
| "time_profile/old_logps": 33.67575552966446, |
| "time_profile/ref_logps": 33.764580024406314, |
| "time_profile/reward": 0.0031769955530762672, |
| "time_profile/text_rollout": 491.6594071071595, |
| "train/gen_step": 8836.0, |
| "und/advantages_abs_mean": 1.008056640625, |
| "und/advantages_max": 2.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.875, |
| "und/advantages_std": 1.0957130193710327, |
| "und/clip_ratio/high_mean": 0.00019178142974851653, |
| "und/clip_ratio/low_mean": 0.0022219074999156874, |
| "und/clip_ratio/region_mean": 0.002413688926026225, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.666015625, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.0, |
| "und/kl": 4483789.968254499, |
| "und/loss": 360.3619811698218, |
| "und/prompt/max_length": 156.0, |
| "und/prompt/mean_length": 156.0, |
| "und/prompt/min_length": 156.0, |
| "und/reward": 1.16015625, |
| "und/reward_std": 1.161067008972168 |
| }, |
| { |
| "epoch": 1.7814251401120895, |
| "grad_norm": 0.1526426076889038, |
| "learning_rate": 1e-05, |
| "loss": 0.0064, |
| "step": 140, |
| "time_profile/curr_logprobs_and_update": 0.26170871233625803, |
| "train/gen_step": 8900.0, |
| "und/clip_ratio/high_mean": 0.0006244859760045074, |
| "und/clip_ratio/low_mean": 0.000251247278356459, |
| "und/clip_ratio/region_mean": 0.0008757332579989452, |
| "und/kl": 0.16707535361638293, |
| "und/loss": 0.0017974353395402431 |
| }, |
| { |
| "epoch": 1.7942353883106485, |
| "grad_norm": 0.04025937244296074, |
| "learning_rate": 1e-05, |
| "loss": 0.0154, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.1015625, |
| "rewards/strict_format_reward_func/std": 0.20275264978408813, |
| "step": 141, |
| "time_profile/curr_logprobs_and_update": 0.26171166240237653, |
| "time_profile/old_logps": 33.693387147039175, |
| "time_profile/ref_logps": 33.75184701196849, |
| "time_profile/reward": 0.00441189669072628, |
| "time_profile/text_rollout": 518.2822579685599, |
| "train/gen_step": 8964.0, |
| "und/advantages_abs_mean": 0.106201171875, |
| "und/advantages_max": 2.125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.375, |
| "und/advantages_std": 0.18686699867248535, |
| "und/clip_ratio/high_mean": 0.0013163000876375008, |
| "und/clip_ratio/low_mean": 0.0009350144391646609, |
| "und/clip_ratio/region_mean": 0.002251314519526204, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.88671875, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.515625, |
| "und/kl": 0.1287980208871886, |
| "und/loss": 0.028064919478310912, |
| "und/prompt/max_length": 313.0, |
| "und/prompt/mean_length": 237.5, |
| "und/prompt/min_length": 162.0, |
| "und/reward": 0.0888671875, |
| "und/reward_std": 0.21538907289505005 |
| }, |
| { |
| "epoch": 1.8070456365092074, |
| "grad_norm": 0.010626863688230515, |
| "learning_rate": 1e-05, |
| "loss": 260079.9062, |
| "step": 142, |
| "time_profile/curr_logprobs_and_update": 0.2617065573576838, |
| "train/gen_step": 9028.0, |
| "und/clip_ratio/high_mean": 5.5803571740398183e-05, |
| "und/clip_ratio/low_mean": 0.0006520278329844587, |
| "und/clip_ratio/region_mean": 0.0007078314047248568, |
| "und/kl": 6501997.801571596, |
| "und/loss": 2068612.8666643738 |
| }, |
| { |
| "epoch": 1.8198558847077662, |
| "grad_norm": 0.0026135630905628204, |
| "learning_rate": 1e-05, |
| "loss": 8.5116, |
| "rewards/correctness_reward_func/mean": 1.0, |
| "rewards/correctness_reward_func/std": 1.0079052448272705, |
| "rewards/strict_format_reward_func/mean": 0.25, |
| "rewards/strict_format_reward_func/std": 0.2519763112068176, |
| "step": 143, |
| "time_profile/curr_logprobs_and_update": 0.26167610530683305, |
| "time_profile/old_logps": 33.65058270841837, |
| "time_profile/ref_logps": 33.7399518340826, |
| "time_profile/reward": 0.004332819953560829, |
| "time_profile/text_rollout": 490.11245891079307, |
| "train/gen_step": 9092.0, |
| "und/advantages_abs_mean": 0.08447265625, |
| "und/advantages_max": 2.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -2.1875, |
| "und/advantages_std": 0.2977820634841919, |
| "und/clip_ratio/high_mean": 0.00011625744446064346, |
| "und/clip_ratio/low_mean": 0.0014596742403227836, |
| "und/clip_ratio/region_mean": 0.0015759316884214059, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.814453125, |
| "und/completion/min_length": 215.0, |
| "und/frac_reward_zero_std": 0.765625, |
| "und/kl": 212.7892216272594, |
| "und/loss": 0.10032683075525028, |
| "und/prompt/max_length": 247.0, |
| "und/prompt/mean_length": 145.5, |
| "und/prompt/min_length": 44.0, |
| "und/reward": 1.240234375, |
| "und/reward_std": 1.2425521612167358 |
| }, |
| { |
| "epoch": 1.8326661329063252, |
| "grad_norm": 0.00504687987267971, |
| "learning_rate": 1e-05, |
| "loss": 0.0149, |
| "step": 144, |
| "time_profile/curr_logprobs_and_update": 0.2615732659032801, |
| "train/gen_step": 9156.0, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 4.882812572759576e-05, |
| "und/clip_ratio/region_mean": 4.882812572759576e-05, |
| "und/kl": 0.372995498473756, |
| "und/loss": 0.006929575064759774 |
| }, |
| { |
| "epoch": 1.8454763811048838, |
| "grad_norm": 0.052012741565704346, |
| "learning_rate": 1e-05, |
| "loss": 0.0058, |
| "rewards/correctness_reward_func/mean": 0.5, |
| "rewards/correctness_reward_func/std": 0.8728715777397156, |
| "rewards/strict_format_reward_func/mean": 0.2265625, |
| "rewards/strict_format_reward_func/std": 0.250866562128067, |
| "step": 145, |
| "time_profile/curr_logprobs_and_update": 0.26171143807005137, |
| "time_profile/old_logps": 33.68571895174682, |
| "time_profile/ref_logps": 33.76929687988013, |
| "time_profile/reward": 0.004742258228361607, |
| "time_profile/text_rollout": 489.8518072050065, |
| "train/gen_step": 9220.0, |
| "und/advantages_abs_mean": 0.578369140625, |
| "und/advantages_max": 2.0625, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.75, |
| "und/advantages_std": 0.7647761702537537, |
| "und/clip_ratio/high_mean": 0.00015467782941414043, |
| "und/clip_ratio/low_mean": 4.542151145869866e-05, |
| "und/clip_ratio/region_mean": 0.0002000993408728391, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.771484375, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.015625, |
| "und/kl": 0.46287906426005065, |
| "und/loss": -0.01604961227064905, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 159.0, |
| "und/prompt/min_length": 156.0, |
| "und/reward": 0.703125, |
| "und/reward_std": 0.9880647659301758 |
| }, |
| { |
| "epoch": 1.8582866293034428, |
| "grad_norm": 0.0647926777601242, |
| "learning_rate": 1e-05, |
| "loss": 13804.5205, |
| "step": 146, |
| "time_profile/curr_logprobs_and_update": 0.26155846660549287, |
| "train/gen_step": 9284.0, |
| "und/clip_ratio/high_mean": 0.00046555879816878587, |
| "und/clip_ratio/low_mean": 0.0007697123000980355, |
| "und/clip_ratio/region_mean": 0.0012352710982668214, |
| "und/kl": 345113.0145757744, |
| "und/loss": 1.7385565766200628 |
| }, |
| { |
| "epoch": 1.8710968775020016, |
| "grad_norm": 6.310885906219482, |
| "learning_rate": 1e-05, |
| "loss": 148005.3906, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.09375, |
| "rewards/strict_format_reward_func/std": 0.19669894874095917, |
| "step": 147, |
| "time_profile/curr_logprobs_and_update": 0.2616866462340113, |
| "time_profile/old_logps": 33.69251477625221, |
| "time_profile/ref_logps": 33.79237876459956, |
| "time_profile/reward": 0.004402017220854759, |
| "time_profile/text_rollout": 491.3923704344779, |
| "train/gen_step": 9348.0, |
| "und/advantages_abs_mean": 0.169677734375, |
| "und/advantages_max": 0.4375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.25, |
| "und/advantages_std": 0.20616121590137482, |
| "und/clip_ratio/high_mean": 0.0011721192167897243, |
| "und/clip_ratio/low_mean": 0.001822430691390764, |
| "und/clip_ratio/region_mean": 0.002994549911818467, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.83984375, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.078125, |
| "und/kl": 3700134.6438976433, |
| "und/loss": 27.951937119380318, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 162.0, |
| "und/prompt/min_length": 162.0, |
| "und/reward": 0.1298828125, |
| "und/reward_std": 0.2194673866033554 |
| }, |
| { |
| "epoch": 1.8839071257005604, |
| "grad_norm": 220.56588745117188, |
| "learning_rate": 1e-05, |
| "loss": 6.8814, |
| "step": 148, |
| "time_profile/curr_logprobs_and_update": 0.26165977913478855, |
| "train/gen_step": 9412.0, |
| "und/clip_ratio/high_mean": 0.0023689054542046506, |
| "und/clip_ratio/low_mean": 0.0010092035663546994, |
| "und/clip_ratio/region_mean": 0.003378109024197329, |
| "und/kl": 68.11535315206856, |
| "und/loss": 2.0457846918769746 |
| }, |
| { |
| "epoch": 1.8967173738991194, |
| "grad_norm": 1.7173937559127808, |
| "learning_rate": 1e-05, |
| "loss": 9899996.0, |
| "rewards/correctness_reward_func/mean": 0.5625, |
| "rewards/correctness_reward_func/std": 0.9063270092010498, |
| "rewards/strict_format_reward_func/mean": 0.265625, |
| "rewards/strict_format_reward_func/std": 0.2514837086200714, |
| "step": 149, |
| "time_profile/curr_logprobs_and_update": 0.2616911970835645, |
| "time_profile/old_logps": 33.67640256509185, |
| "time_profile/ref_logps": 33.75797679182142, |
| "time_profile/reward": 0.004289323464035988, |
| "time_profile/text_rollout": 474.0713466331363, |
| "train/gen_step": 9476.0, |
| "und/advantages_abs_mean": 0.607421875, |
| "und/advantages_max": 2.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -2.1875, |
| "und/advantages_std": 0.7971558570861816, |
| "und/clip_ratio/high_mean": 0.0029124355169187766, |
| "und/clip_ratio/low_mean": 0.0017562373941473197, |
| "und/clip_ratio/region_mean": 0.004668672914704075, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.71875, |
| "und/completion/min_length": 251.0, |
| "und/frac_reward_zero_std": 0.015625, |
| "und/kl": 247499896.9961213, |
| "und/loss": 437175.3625415751, |
| "und/prompt/max_length": 156.0, |
| "und/prompt/mean_length": 105.0, |
| "und/prompt/min_length": 54.0, |
| "und/reward": 0.8623046875, |
| "und/reward_std": 1.0626578330993652 |
| }, |
| { |
| "epoch": 1.909527622097678, |
| "grad_norm": 0.011920370161533356, |
| "learning_rate": 1e-05, |
| "loss": 0.0134, |
| "step": 150, |
| "time_profile/curr_logprobs_and_update": 0.26148566768097226, |
| "train/gen_step": 9540.0, |
| "und/clip_ratio/high_mean": 0.000891603427589871, |
| "und/clip_ratio/low_mean": 0.00019111833535134792, |
| "und/clip_ratio/region_mean": 0.001082721762941219, |
| "und/kl": 0.3309498258458916, |
| "und/loss": 0.0035182901192456484 |
| }, |
| { |
| "epoch": 1.922337870296237, |
| "grad_norm": 0.04238224774599075, |
| "learning_rate": 1e-05, |
| "loss": -0.0036, |
| "rewards/correctness_reward_func/mean": 0.5625, |
| "rewards/correctness_reward_func/std": 0.9063270092010498, |
| "rewards/strict_format_reward_func/mean": 0.3671875, |
| "rewards/strict_format_reward_func/std": 0.22257846593856812, |
| "step": 151, |
| "time_profile/curr_logprobs_and_update": 0.2615245400520507, |
| "time_profile/old_logps": 33.659803338348866, |
| "time_profile/ref_logps": 33.75399888958782, |
| "time_profile/reward": 0.00458063930273056, |
| "time_profile/text_rollout": 471.09525199607015, |
| "train/gen_step": 9604.0, |
| "und/advantages_abs_mean": 0.615234375, |
| "und/advantages_max": 2.0625, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.5, |
| "und/advantages_std": 0.7951586842536926, |
| "und/clip_ratio/high_mean": 0.0020844707141804975, |
| "und/clip_ratio/low_mean": 0.0001922483897942584, |
| "und/clip_ratio/region_mean": 0.002276719100336777, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.716796875, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.015625, |
| "und/kl": 0.05341187393059954, |
| "und/loss": 0.009631790686398745, |
| "und/prompt/max_length": 156.0, |
| "und/prompt/mean_length": 99.5, |
| "und/prompt/min_length": 43.0, |
| "und/reward": 0.8447265625, |
| "und/reward_std": 0.9140197038650513 |
| }, |
| { |
| "epoch": 1.9351481184947958, |
| "grad_norm": 0.06001448258757591, |
| "learning_rate": 1e-05, |
| "loss": 0.0095, |
| "step": 152, |
| "time_profile/curr_logprobs_and_update": 0.261558934187633, |
| "train/gen_step": 9668.0, |
| "und/clip_ratio/high_mean": 0.0005441326029540505, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0005441326029540505, |
| "und/kl": 0.2372699785628356, |
| "und/loss": 0.0034325564286064036 |
| }, |
| { |
| "epoch": 1.9479583666933546, |
| "grad_norm": 0.026463812217116356, |
| "learning_rate": 1e-05, |
| "loss": 0.0137, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.1875, |
| "rewards/strict_format_reward_func/std": 0.24397502839565277, |
| "step": 153, |
| "time_profile/curr_logprobs_and_update": 0.2616782131517539, |
| "time_profile/old_logps": 33.68901115562767, |
| "time_profile/ref_logps": 33.789112674072385, |
| "time_profile/reward": 0.004856881685554981, |
| "time_profile/text_rollout": 476.91363042313606, |
| "train/gen_step": 9732.0, |
| "und/advantages_abs_mean": 0.197509765625, |
| "und/advantages_max": 0.4375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.375, |
| "und/advantages_std": 0.22242768108844757, |
| "und/clip_ratio/high_mean": 0.0009648025552451145, |
| "und/clip_ratio/low_mean": 6.734913768013939e-05, |
| "und/clip_ratio/region_mean": 0.0010321516929252539, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.77734375, |
| "und/completion/min_length": 252.0, |
| "und/frac_reward_zero_std": 0.03125, |
| "und/kl": 0.22904289467260242, |
| "und/loss": -0.018844104985873855, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 111.5, |
| "und/prompt/min_length": 61.0, |
| "und/reward": 0.1689453125, |
| "und/reward_std": 0.2367268204689026 |
| }, |
| { |
| "epoch": 1.9607686148919137, |
| "grad_norm": 152.77015686035156, |
| "learning_rate": 1e-05, |
| "loss": 0.5685, |
| "step": 154, |
| "time_profile/curr_logprobs_and_update": 0.26678641652688384, |
| "train/gen_step": 9796.0, |
| "und/clip_ratio/high_mean": 0.0005597944764303975, |
| "und/clip_ratio/low_mean": 0.00025797137641347945, |
| "und/clip_ratio/region_mean": 0.0008177658528438769, |
| "und/kl": 14.211894012492849, |
| "und/loss": 0.004249518504366279 |
| }, |
| { |
| "epoch": 1.9735788630904723, |
| "grad_norm": 0.010165955871343613, |
| "learning_rate": 1e-05, |
| "loss": 0.015, |
| "rewards/correctness_reward_func/mean": 0.15625, |
| "rewards/correctness_reward_func/std": 0.5409794449806213, |
| "rewards/strict_format_reward_func/mean": 0.3359375, |
| "rewards/strict_format_reward_func/std": 0.2366211861371994, |
| "step": 155, |
| "time_profile/curr_logprobs_and_update": 0.2616322033718461, |
| "time_profile/old_logps": 33.655117710120976, |
| "time_profile/ref_logps": 33.740064659155905, |
| "time_profile/reward": 0.004281140863895416, |
| "time_profile/text_rollout": 472.37845607008785, |
| "train/gen_step": 9860.0, |
| "und/advantages_abs_mean": 0.363037109375, |
| "und/advantages_max": 1.8125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.5, |
| "und/advantages_std": 0.5413435697555542, |
| "und/clip_ratio/high_mean": 0.0021703395432268735, |
| "und/clip_ratio/low_mean": 0.0006125769177742768, |
| "und/clip_ratio/region_mean": 0.002782916464639129, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.7578125, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.109375, |
| "und/kl": 0.2890739535650937, |
| "und/loss": 0.0022308455081656575, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 103.0, |
| "und/prompt/min_length": 44.0, |
| "und/reward": 0.53515625, |
| "und/reward_std": 0.6929434537887573 |
| }, |
| { |
| "epoch": 1.9863891112890313, |
| "grad_norm": 0.033500850200653076, |
| "learning_rate": 1e-05, |
| "loss": 0.0053, |
| "step": 156, |
| "time_profile/curr_logprobs_and_update": 0.2615450029115891, |
| "train/gen_step": 9924.0, |
| "und/clip_ratio/high_mean": 0.0014465084786934312, |
| "und/clip_ratio/low_mean": 0.00015304287444450893, |
| "und/clip_ratio/region_mean": 0.0015995513422240037, |
| "und/kl": 0.12355754713644274, |
| "und/loss": 0.017674150729817484 |
| }, |
| { |
| "epoch": 1.99919935948759, |
| "grad_norm": 0.011605536565184593, |
| "learning_rate": 1e-05, |
| "loss": 0.0113, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.09375, |
| "rewards/strict_format_reward_func/std": 0.19669894874095917, |
| "step": 157, |
| "time_profile/curr_logprobs_and_update": 0.26163526308664586, |
| "time_profile/old_logps": 33.68655322957784, |
| "time_profile/ref_logps": 33.78236538451165, |
| "time_profile/reward": 0.0044804466888308525, |
| "time_profile/text_rollout": 518.2296011028811, |
| "train/gen_step": 9988.0, |
| "und/advantages_abs_mean": 0.12353515625, |
| "und/advantages_max": 2.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.375, |
| "und/advantages_std": 0.19876083731651306, |
| "und/clip_ratio/high_mean": 0.001845829377998598, |
| "und/clip_ratio/low_mean": 0.000797167816926958, |
| "und/clip_ratio/region_mean": 0.002642997173097683, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.859375, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.453125, |
| "und/kl": 0.2580548677360639, |
| "und/loss": 0.016669023714030118, |
| "und/prompt/max_length": 313.0, |
| "und/prompt/mean_length": 237.5, |
| "und/prompt/min_length": 162.0, |
| "und/reward": 0.109375, |
| "und/reward_std": 0.22933153808116913 |
| }, |
| { |
| "epoch": 2.0, |
| "grad_norm": 0.008261620998382568, |
| "learning_rate": 1e-05, |
| "loss": -0.0022, |
| "step": 158, |
| "time_profile/curr_logprobs_and_update": 0.2615143822040409, |
| "train/gen_step": 9992.0, |
| "und/clip_ratio/high_mean": 0.0008680555620230734, |
| "und/clip_ratio/low_mean": 0.0017684109043329954, |
| "und/clip_ratio/region_mean": 0.002636466466356069, |
| "und/kl": 0.0970268496312201, |
| "und/loss": -0.06052475795149803 |
| }, |
| { |
| "epoch": 2.012810248198559, |
| "grad_norm": 0.007182754576206207, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "step": 159, |
| "time_profile/curr_logprobs_and_update": 0.2617839087470202, |
| "train/gen_step": 10056.0, |
| "und/clip_ratio/high_mean": 0.0009972724728868343, |
| "und/clip_ratio/low_mean": 0.00033537117633386515, |
| "und/clip_ratio/region_mean": 0.0013326436528586783, |
| "und/kl": 0.06421022860740777, |
| "und/loss": 0.0018333349892145634 |
| }, |
| { |
| "epoch": 2.0256204963971176, |
| "grad_norm": 0.00574113754555583, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "rewards/correctness_reward_func/mean": 0.5625, |
| "rewards/correctness_reward_func/std": 0.9063270092010498, |
| "rewards/strict_format_reward_func/mean": 0.34375, |
| "rewards/strict_format_reward_func/std": 0.233588308095932, |
| "step": 160, |
| "time_profile/curr_logprobs_and_update": 0.2617837716679787, |
| "time_profile/old_logps": 33.679151609539986, |
| "time_profile/ref_logps": 33.74794678296894, |
| "time_profile/reward": 0.004701966419816017, |
| "time_profile/text_rollout": 472.4926246367395, |
| "train/gen_step": 10120.0, |
| "und/advantages_abs_mean": 0.545654296875, |
| "und/advantages_max": 1.625, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -2.1875, |
| "und/advantages_std": 0.6981857419013977, |
| "und/clip_ratio/high_mean": 0.0016045431657403242, |
| "und/clip_ratio/low_mean": 0.00043878793803742155, |
| "und/clip_ratio/region_mean": 0.0020433310928638093, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.75, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.0, |
| "und/kl": 0.07028049726795871, |
| "und/loss": 0.007069828358226005, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 103.5, |
| "und/prompt/min_length": 45.0, |
| "und/reward": 0.9853515625, |
| "und/reward_std": 1.0728275775909424 |
| }, |
| { |
| "epoch": 2.0384307445956766, |
| "grad_norm": 0.08203817158937454, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "step": 161, |
| "time_profile/curr_logprobs_and_update": 0.2615642463497352, |
| "train/gen_step": 10184.0, |
| "und/clip_ratio/high_mean": 0.00023133041395340115, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.00023133041395340115, |
| "und/kl": 0.06791361824434716, |
| "und/loss": 0.0056212409399449825 |
| }, |
| { |
| "epoch": 2.051240992794235, |
| "grad_norm": 0.09558697044849396, |
| "learning_rate": 1e-05, |
| "loss": 0.0037, |
| "rewards/correctness_reward_func/mean": 0.3125, |
| "rewards/correctness_reward_func/std": 0.7319250702857971, |
| "rewards/strict_format_reward_func/mean": 0.328125, |
| "rewards/strict_format_reward_func/std": 0.23935678601264954, |
| "step": 162, |
| "time_profile/curr_logprobs_and_update": 0.26180726967868395, |
| "time_profile/old_logps": 33.68221877235919, |
| "time_profile/ref_logps": 33.787430736236274, |
| "time_profile/reward": 0.004500852897763252, |
| "time_profile/text_rollout": 472.37788531929255, |
| "train/gen_step": 10248.0, |
| "und/advantages_abs_mean": 0.521484375, |
| "und/advantages_max": 2.0, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.625, |
| "und/advantages_std": 0.6933073997497559, |
| "und/clip_ratio/high_mean": 0.0011127722791570704, |
| "und/clip_ratio/low_mean": 0.001079595233022701, |
| "und/clip_ratio/region_mean": 0.0021923675085417926, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.630859375, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.0, |
| "und/kl": 0.10435478831641376, |
| "und/loss": 0.00502297020284459, |
| "und/prompt/max_length": 156.0, |
| "und/prompt/mean_length": 105.5, |
| "und/prompt/min_length": 55.0, |
| "und/reward": 0.6328125, |
| "und/reward_std": 0.8416374921798706 |
| }, |
| { |
| "epoch": 2.0640512409927942, |
| "grad_norm": 0.04183354601264, |
| "learning_rate": 1e-05, |
| "loss": 0.0066, |
| "step": 163, |
| "time_profile/curr_logprobs_and_update": 0.26139140441955533, |
| "train/gen_step": 10312.0, |
| "und/clip_ratio/high_mean": 0.00019865745343849994, |
| "und/clip_ratio/low_mean": 0.00014263017146731727, |
| "und/clip_ratio/region_mean": 0.0003412876249058172, |
| "und/kl": 0.16455070252413861, |
| "und/loss": 0.002166528080124408 |
| }, |
| { |
| "epoch": 2.0768614891913533, |
| "grad_norm": 0.009151386097073555, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "rewards/correctness_reward_func/mean": 0.0625, |
| "rewards/correctness_reward_func/std": 0.35073620080947876, |
| "rewards/strict_format_reward_func/mean": 0.34375, |
| "rewards/strict_format_reward_func/std": 0.233588308095932, |
| "step": 164, |
| "time_profile/curr_logprobs_and_update": 0.26157606378546916, |
| "time_profile/old_logps": 33.671810798346996, |
| "time_profile/ref_logps": 33.7788968719542, |
| "time_profile/reward": 0.004629682749509811, |
| "time_profile/text_rollout": 472.72225496545434, |
| "train/gen_step": 10376.0, |
| "und/advantages_abs_mean": 0.18994140625, |
| "und/advantages_max": 1.75, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.5, |
| "und/advantages_std": 0.3282596468925476, |
| "und/clip_ratio/high_mean": 0.0003861400473397225, |
| "und/clip_ratio/low_mean": 0.00010074145757243969, |
| "und/clip_ratio/region_mean": 0.0004868815049121622, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.779296875, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.3125, |
| "und/kl": 0.06336885890777921, |
| "und/loss": 0.0024055056273937225, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 102.5, |
| "und/prompt/min_length": 43.0, |
| "und/reward": 0.392578125, |
| "und/reward_std": 0.39958831667900085 |
| }, |
| { |
| "epoch": 2.089671737389912, |
| "grad_norm": 0.0061566890217363834, |
| "learning_rate": 1e-05, |
| "loss": 0.0062, |
| "step": 165, |
| "time_profile/curr_logprobs_and_update": 0.26149553031427786, |
| "train/gen_step": 10440.0, |
| "und/clip_ratio/high_mean": 0.0001600477480678819, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0001600477480678819, |
| "und/kl": 0.154379196363152, |
| "und/loss": 0.0021597322743218683 |
| }, |
| { |
| "epoch": 2.102481985588471, |
| "grad_norm": 0.06113005802035332, |
| "learning_rate": 1e-05, |
| "loss": 0.0048, |
| "rewards/correctness_reward_func/mean": 0.78125, |
| "rewards/correctness_reward_func/std": 0.983494758605957, |
| "rewards/strict_format_reward_func/mean": 0.375, |
| "rewards/strict_format_reward_func/std": 0.2182178944349289, |
| "step": 166, |
| "time_profile/curr_logprobs_and_update": 0.2617073591245571, |
| "time_profile/old_logps": 33.64431105554104, |
| "time_profile/ref_logps": 33.74050881713629, |
| "time_profile/reward": 0.004514393396675587, |
| "time_profile/text_rollout": 454.8430243739858, |
| "train/gen_step": 10504.0, |
| "und/advantages_abs_mean": 0.470703125, |
| "und/advantages_max": 1.5, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.9375, |
| "und/advantages_std": 0.6300917863845825, |
| "und/clip_ratio/high_mean": 0.0004267864424036816, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0004267864424036816, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.6953125, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.046875, |
| "und/kl": 0.11857564687670674, |
| "und/loss": 0.003597188484036451, |
| "und/prompt/max_length": 50.0, |
| "und/prompt/mean_length": 50.0, |
| "und/prompt/min_length": 50.0, |
| "und/reward": 1.08984375, |
| "und/reward_std": 1.0457885265350342 |
| }, |
| { |
| "epoch": 2.1152922337870295, |
| "grad_norm": 0.03352009132504463, |
| "learning_rate": 1e-05, |
| "loss": 0.006, |
| "step": 167, |
| "time_profile/curr_logprobs_and_update": 0.26128566317493096, |
| "train/gen_step": 10568.0, |
| "und/clip_ratio/high_mean": 0.0015908839777694084, |
| "und/clip_ratio/low_mean": 4.542151145869866e-05, |
| "und/clip_ratio/region_mean": 0.001636305489228107, |
| "und/kl": 0.11126315701403655, |
| "und/loss": 0.004314804737987288 |
| }, |
| { |
| "epoch": 2.1281024819855885, |
| "grad_norm": 0.02977151609957218, |
| "learning_rate": 1e-05, |
| "loss": 0.0109, |
| "rewards/correctness_reward_func/mean": 0.03125, |
| "rewards/correctness_reward_func/std": 0.25, |
| "rewards/strict_format_reward_func/mean": 0.25, |
| "rewards/strict_format_reward_func/std": 0.2519763112068176, |
| "step": 168, |
| "time_profile/curr_logprobs_and_update": 0.2614454288268462, |
| "time_profile/old_logps": 33.630763033404946, |
| "time_profile/ref_logps": 33.74494473170489, |
| "time_profile/reward": 0.004755772650241852, |
| "time_profile/text_rollout": 487.7817754801363, |
| "train/gen_step": 10632.0, |
| "und/advantages_abs_mean": 0.058349609375, |
| "und/advantages_max": 2.125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.5, |
| "und/advantages_std": 0.20196563005447388, |
| "und/clip_ratio/high_mean": 0.0013399315866990946, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0013399315866990946, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.71484375, |
| "und/completion/min_length": 221.0, |
| "und/frac_reward_zero_std": 0.703125, |
| "und/kl": 0.047248675022274256, |
| "und/loss": 0.01837195276038983, |
| "und/prompt/max_length": 247.0, |
| "und/prompt/mean_length": 149.0, |
| "und/prompt/min_length": 51.0, |
| "und/reward": 0.2822265625, |
| "und/reward_std": 0.30478331446647644 |
| }, |
| { |
| "epoch": 2.1409127301841475, |
| "grad_norm": 0.008049830794334412, |
| "learning_rate": 1e-05, |
| "loss": 0.0037, |
| "step": 169, |
| "time_profile/curr_logprobs_and_update": 0.26160103428992443, |
| "train/gen_step": 10696.0, |
| "und/clip_ratio/high_mean": 0.00010557432688074186, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.00010557432688074186, |
| "und/kl": 0.09378908702638, |
| "und/loss": 0.002309132060418051 |
| }, |
| { |
| "epoch": 2.153722978382706, |
| "grad_norm": 0.03837805986404419, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "rewards/correctness_reward_func/mean": 0.71875, |
| "rewards/correctness_reward_func/std": 0.9672207236289978, |
| "rewards/strict_format_reward_func/mean": 0.40625, |
| "rewards/strict_format_reward_func/std": 0.19669894874095917, |
| "step": 170, |
| "time_profile/curr_logprobs_and_update": 0.2617557874182239, |
| "time_profile/old_logps": 33.67024072445929, |
| "time_profile/ref_logps": 33.74819596670568, |
| "time_profile/reward": 0.004233397543430328, |
| "time_profile/text_rollout": 488.64935347717255, |
| "train/gen_step": 10760.0, |
| "und/advantages_abs_mean": 0.906982421875, |
| "und/advantages_max": 2.0, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.9375, |
| "und/advantages_std": 1.0159564018249512, |
| "und/clip_ratio/high_mean": 0.00015850567069719546, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.00015850567069719546, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.59375, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.0, |
| "und/kl": 0.0635057357721962, |
| "und/loss": 0.00192016769960901, |
| "und/prompt/max_length": 156.0, |
| "und/prompt/mean_length": 156.0, |
| "und/prompt/min_length": 156.0, |
| "und/reward": 1.123046875, |
| "und/reward_std": 1.0842657089233398 |
| }, |
| { |
| "epoch": 2.166533226581265, |
| "grad_norm": 0.04740050062537193, |
| "learning_rate": 1e-05, |
| "loss": 0.0057, |
| "step": 171, |
| "time_profile/curr_logprobs_and_update": 0.2614927099493798, |
| "train/gen_step": 10824.0, |
| "und/clip_ratio/high_mean": 0.00019523592709447257, |
| "und/clip_ratio/low_mean": 5.278716344037093e-05, |
| "und/clip_ratio/region_mean": 0.0002480230905348435, |
| "und/kl": 0.14967676892410964, |
| "und/loss": 0.002013998571783304 |
| }, |
| { |
| "epoch": 2.1793434747798237, |
| "grad_norm": 0.03162504360079765, |
| "learning_rate": 1e-05, |
| "loss": 0.0032, |
| "rewards/correctness_reward_func/mean": 0.09375, |
| "rewards/correctness_reward_func/std": 0.42608407139778137, |
| "rewards/strict_format_reward_func/mean": 0.2734375, |
| "rewards/strict_format_reward_func/std": 0.250866562128067, |
| "step": 172, |
| "time_profile/curr_logprobs_and_update": 0.2618536733352812, |
| "time_profile/old_logps": 33.67922029271722, |
| "time_profile/ref_logps": 33.76980642415583, |
| "time_profile/reward": 0.004333090037107468, |
| "time_profile/text_rollout": 473.33523644600064, |
| "train/gen_step": 10888.0, |
| "und/advantages_abs_mean": 0.21728515625, |
| "und/advantages_max": 2.0, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.6875, |
| "und/advantages_std": 0.2858365774154663, |
| "und/clip_ratio/high_mean": 0.0005348777740437072, |
| "und/clip_ratio/low_mean": 0.0002211474893556442, |
| "und/clip_ratio/region_mean": 0.0007560252670373302, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.728515625, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.046875, |
| "und/kl": 0.08075993752572685, |
| "und/loss": 0.00289958156645298, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 107.5, |
| "und/prompt/min_length": 53.0, |
| "und/reward": 0.27734375, |
| "und/reward_std": 0.31782403588294983 |
| }, |
| { |
| "epoch": 2.1921537229783827, |
| "grad_norm": 0.011727745644748211, |
| "learning_rate": 1e-05, |
| "loss": 0.0019, |
| "step": 173, |
| "time_profile/curr_logprobs_and_update": 0.2616145513165975, |
| "train/gen_step": 10952.0, |
| "und/clip_ratio/high_mean": 6.103515625e-05, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 6.103515625e-05, |
| "und/kl": 0.04787289367232006, |
| "und/loss": 0.001894820132292807 |
| }, |
| { |
| "epoch": 2.2049639711769418, |
| "grad_norm": 0.008552194572985172, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.2890625, |
| "rewards/strict_format_reward_func/std": 0.24888142943382263, |
| "step": 174, |
| "time_profile/curr_logprobs_and_update": 0.26172452403989155, |
| "time_profile/old_logps": 33.6549604870379, |
| "time_profile/ref_logps": 33.75101667176932, |
| "time_profile/reward": 0.004562413319945335, |
| "time_profile/text_rollout": 455.8837287509814, |
| "train/gen_step": 11016.0, |
| "und/advantages_abs_mean": 0.2109375, |
| "und/advantages_max": 0.4375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.4375, |
| "und/advantages_std": 0.22986425459384918, |
| "und/clip_ratio/high_mean": 0.00026825524037121795, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.00026825524037121795, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.73046875, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.015625, |
| "und/kl": 0.054617739195236936, |
| "und/loss": 0.0014219468575902283, |
| "und/prompt/max_length": 58.0, |
| "und/prompt/mean_length": 55.5, |
| "und/prompt/min_length": 53.0, |
| "und/reward": 0.28515625, |
| "und/reward_std": 0.24775780737400055 |
| }, |
| { |
| "epoch": 2.2177742193755003, |
| "grad_norm": 0.011380493640899658, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "step": 175, |
| "time_profile/curr_logprobs_and_update": 0.2613375957298558, |
| "train/gen_step": 11080.0, |
| "und/clip_ratio/high_mean": 0.0009292362919950392, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0009292362919950392, |
| "und/kl": 0.053230694044032134, |
| "und/loss": 0.001982355781365186 |
| }, |
| { |
| "epoch": 2.2305844675740594, |
| "grad_norm": 0.03694207966327667, |
| "learning_rate": 1e-05, |
| "loss": 0.0018, |
| "rewards/correctness_reward_func/mean": 0.625, |
| "rewards/correctness_reward_func/std": 0.934353232383728, |
| "rewards/strict_format_reward_func/mean": 0.1640625, |
| "rewards/strict_format_reward_func/std": 0.2366211861371994, |
| "step": 176, |
| "time_profile/curr_logprobs_and_update": 0.2614654813223751, |
| "time_profile/old_logps": 33.645203595981, |
| "time_profile/ref_logps": 33.76335656829178, |
| "time_profile/reward": 0.004439213313162327, |
| "time_profile/text_rollout": 517.072899880819, |
| "train/gen_step": 11144.0, |
| "und/advantages_abs_mean": 0.5068359375, |
| "und/advantages_max": 2.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -2.1875, |
| "und/advantages_std": 0.777386486530304, |
| "und/clip_ratio/high_mean": 0.001711124616122106, |
| "und/clip_ratio/low_mean": 9.192448851536028e-05, |
| "und/clip_ratio/region_mean": 0.0018030491046374664, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.78515625, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.34375, |
| "und/kl": 0.04419090352894273, |
| "und/loss": 0.001942948205396533, |
| "und/prompt/max_length": 313.0, |
| "und/prompt/mean_length": 234.5, |
| "und/prompt/min_length": 156.0, |
| "und/reward": 0.90234375, |
| "und/reward_std": 1.177177906036377 |
| }, |
| { |
| "epoch": 2.243394715772618, |
| "grad_norm": 0.04087910056114197, |
| "learning_rate": 1e-05, |
| "loss": 0.025, |
| "step": 177, |
| "time_profile/curr_logprobs_and_update": 0.2616091024246998, |
| "train/gen_step": 11208.0, |
| "und/clip_ratio/high_mean": 0.0001443542423658073, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0001443542423658073, |
| "und/kl": 0.6302279950177763, |
| "und/loss": 0.0017156366041035653 |
| }, |
| { |
| "epoch": 2.256204963971177, |
| "grad_norm": 0.06869915127754211, |
| "learning_rate": 1e-05, |
| "loss": 0.0761, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.1015625, |
| "rewards/strict_format_reward_func/std": 0.20275264978408813, |
| "step": 178, |
| "time_profile/curr_logprobs_and_update": 0.26177879216265865, |
| "time_profile/old_logps": 33.71037639584392, |
| "time_profile/ref_logps": 33.78417940158397, |
| "time_profile/reward": 0.004494715481996536, |
| "time_profile/text_rollout": 518.1308631561697, |
| "train/gen_step": 11272.0, |
| "und/advantages_abs_mean": 0.1259765625, |
| "und/advantages_max": 2.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.5625, |
| "und/advantages_std": 0.23105835914611816, |
| "und/clip_ratio/high_mean": 0.0004932920819555875, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0004932920819555875, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.859375, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.484375, |
| "und/kl": 1.9037787910492625, |
| "und/loss": 0.41365190602118673, |
| "und/prompt/max_length": 313.0, |
| "und/prompt/mean_length": 237.5, |
| "und/prompt/min_length": 162.0, |
| "und/reward": 0.1162109375, |
| "und/reward_std": 0.26848655939102173 |
| }, |
| { |
| "epoch": 2.2690152121697356, |
| "grad_norm": 0.00810084119439125, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "step": 179, |
| "time_profile/curr_logprobs_and_update": 0.2615918532537762, |
| "train/gen_step": 11336.0, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/kl": 0.054657230590237305, |
| "und/loss": 0.0015159287313508685 |
| }, |
| { |
| "epoch": 2.2818254603682946, |
| "grad_norm": 0.010132861323654652, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.234375, |
| "rewards/strict_format_reward_func/std": 0.2514837086200714, |
| "step": 180, |
| "time_profile/curr_logprobs_and_update": 0.2618380212079501, |
| "time_profile/old_logps": 33.66110870428383, |
| "time_profile/ref_logps": 33.76540139503777, |
| "time_profile/reward": 0.004269592463970184, |
| "time_profile/text_rollout": 473.37367711402476, |
| "train/gen_step": 11400.0, |
| "und/advantages_abs_mean": 0.229736328125, |
| "und/advantages_max": 2.0, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.5625, |
| "und/advantages_std": 0.27835747599601746, |
| "und/clip_ratio/high_mean": 0.000102588386653224, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.000102588386653224, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.732421875, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.03125, |
| "und/kl": 0.07371564902132377, |
| "und/loss": 0.0017190172418395377, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 108.0, |
| "und/prompt/min_length": 54.0, |
| "und/reward": 0.2607421875, |
| "und/reward_std": 0.29655858874320984 |
| }, |
| { |
| "epoch": 2.2946357085668536, |
| "grad_norm": 0.008120850659906864, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "step": 181, |
| "time_profile/curr_logprobs_and_update": 0.26148502981232014, |
| "train/gen_step": 11464.0, |
| "und/clip_ratio/high_mean": 0.0003160519918310456, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0003160519918310456, |
| "und/kl": 0.06285516830394045, |
| "und/loss": 0.0015067223123139684 |
| }, |
| { |
| "epoch": 2.307445956765412, |
| "grad_norm": 0.009181732311844826, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "rewards/correctness_reward_func/mean": 0.125, |
| "rewards/correctness_reward_func/std": 0.48795005679130554, |
| "rewards/strict_format_reward_func/mean": 0.328125, |
| "rewards/strict_format_reward_func/std": 0.23935678601264954, |
| "step": 182, |
| "time_profile/curr_logprobs_and_update": 0.2617022335034562, |
| "time_profile/old_logps": 33.64694704860449, |
| "time_profile/ref_logps": 33.735723273828626, |
| "time_profile/reward": 0.0047216713428497314, |
| "time_profile/text_rollout": 472.53653713781387, |
| "train/gen_step": 11528.0, |
| "und/advantages_abs_mean": 0.240478515625, |
| "und/advantages_max": 1.8125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.6875, |
| "und/advantages_std": 0.4047631025314331, |
| "und/clip_ratio/high_mean": 0.00048385561603936367, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.00048385561603936367, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.69140625, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.21875, |
| "und/kl": 0.0520571748202201, |
| "und/loss": 0.002413793592950242, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 104.0, |
| "und/prompt/min_length": 46.0, |
| "und/reward": 0.4287109375, |
| "und/reward_std": 0.46533018350601196 |
| }, |
| { |
| "epoch": 2.3202562049639712, |
| "grad_norm": 0.02421620301902294, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "step": 183, |
| "time_profile/curr_logprobs_and_update": 0.2613777341321111, |
| "train/gen_step": 11592.0, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0, |
| "und/kl": 0.0602908979926724, |
| "und/loss": 0.002848730291134416 |
| }, |
| { |
| "epoch": 2.3330664531625303, |
| "grad_norm": 0.024613449349999428, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.2734375, |
| "rewards/strict_format_reward_func/std": 0.250866562128067, |
| "step": 184, |
| "time_profile/curr_logprobs_and_update": 0.261491294571897, |
| "time_profile/old_logps": 33.67325418256223, |
| "time_profile/ref_logps": 33.7762045590207, |
| "time_profile/reward": 0.004586059600114822, |
| "time_profile/text_rollout": 473.8289050068706, |
| "train/gen_step": 11656.0, |
| "und/advantages_abs_mean": 0.174560546875, |
| "und/advantages_max": 2.0625, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.625, |
| "und/advantages_std": 0.25663885474205017, |
| "und/clip_ratio/high_mean": 0.0, |
| "und/clip_ratio/low_mean": 5.7444853155175224e-05, |
| "und/clip_ratio/region_mean": 5.7444853155175224e-05, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.603515625, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.15625, |
| "und/kl": 0.06306225908338092, |
| "und/loss": 0.00225927818718219, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 109.0, |
| "und/prompt/min_length": 56.0, |
| "und/reward": 0.3388671875, |
| "und/reward_std": 0.2899373769760132 |
| }, |
| { |
| "epoch": 2.345876701361089, |
| "grad_norm": 0.0077704801224172115, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "step": 185, |
| "time_profile/curr_logprobs_and_update": 0.2616247042315081, |
| "train/gen_step": 11720.0, |
| "und/clip_ratio/high_mean": 0.00011574074233067222, |
| "und/clip_ratio/low_mean": 8.877841173671186e-05, |
| "und/clip_ratio/region_mean": 0.00020451915406738408, |
| "und/kl": 0.06688823399599642, |
| "und/loss": 0.003165125963278115 |
| }, |
| { |
| "epoch": 2.358686949559648, |
| "grad_norm": 0.049835316836833954, |
| "learning_rate": 1e-05, |
| "loss": 0.0035, |
| "rewards/correctness_reward_func/mean": 1.0625, |
| "rewards/correctness_reward_func/std": 1.0059348344802856, |
| "rewards/strict_format_reward_func/mean": 0.3984375, |
| "rewards/strict_format_reward_func/std": 0.20275264978408813, |
| "step": 186, |
| "time_profile/curr_logprobs_and_update": 0.26167551841354, |
| "time_profile/old_logps": 33.6585817206651, |
| "time_profile/ref_logps": 33.735864114016294, |
| "time_profile/reward": 0.00426170788705349, |
| "time_profile/text_rollout": 470.94582207594067, |
| "train/gen_step": 11784.0, |
| "und/advantages_abs_mean": 0.705078125, |
| "und/advantages_max": 2.0, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -2.1875, |
| "und/advantages_std": 0.8917018175125122, |
| "und/clip_ratio/high_mean": 0.00022136736151878722, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.00022136736151878722, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.521484375, |
| "und/completion/min_length": 220.0, |
| "und/frac_reward_zero_std": 0.09375, |
| "und/kl": 0.08871068802545778, |
| "und/loss": 0.0025096102617681026, |
| "und/prompt/max_length": 156.0, |
| "und/prompt/mean_length": 99.0, |
| "und/prompt/min_length": 42.0, |
| "und/reward": 1.44921875, |
| "und/reward_std": 1.124082326889038 |
| }, |
| { |
| "epoch": 2.3714971977582064, |
| "grad_norm": 0.07230977714061737, |
| "learning_rate": 1e-05, |
| "loss": 0.0038, |
| "step": 187, |
| "time_profile/curr_logprobs_and_update": 0.2613866949104704, |
| "train/gen_step": 11848.0, |
| "und/clip_ratio/high_mean": 0.00014307776291389018, |
| "und/clip_ratio/low_mean": 6.300403038039804e-05, |
| "und/clip_ratio/region_mean": 0.00020608179329428822, |
| "und/kl": 0.097599101572996, |
| "und/loss": 0.0026581829088172526 |
| }, |
| { |
| "epoch": 2.3843074459567655, |
| "grad_norm": 0.09674729406833649, |
| "learning_rate": 1e-05, |
| "loss": 0.0043, |
| "rewards/correctness_reward_func/mean": 0.0, |
| "rewards/correctness_reward_func/std": 0.0, |
| "rewards/strict_format_reward_func/mean": 0.1796875, |
| "rewards/strict_format_reward_func/std": 0.24180518090724945, |
| "step": 188, |
| "time_profile/curr_logprobs_and_update": 0.26152521098265424, |
| "time_profile/old_logps": 33.65347420424223, |
| "time_profile/ref_logps": 33.748919220641255, |
| "time_profile/reward": 0.004438244737684727, |
| "time_profile/text_rollout": 473.7467489670962, |
| "train/gen_step": 11912.0, |
| "und/advantages_abs_mean": 0.2021484375, |
| "und/advantages_max": 0.4375, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.375, |
| "und/advantages_std": 0.22502446174621582, |
| "und/clip_ratio/high_mean": 0.00026816331228474155, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.00026816331228474155, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.77734375, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.046875, |
| "und/kl": 0.10714392748195678, |
| "und/loss": 0.0022139963675726904, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 109.5, |
| "und/prompt/min_length": 57.0, |
| "und/reward": 0.1875, |
| "und/reward_std": 0.2422981858253479 |
| }, |
| { |
| "epoch": 2.397117694155324, |
| "grad_norm": 0.013547060079872608, |
| "learning_rate": 1e-05, |
| "loss": 0.0071, |
| "step": 189, |
| "time_profile/curr_logprobs_and_update": 0.2615761066117557, |
| "train/gen_step": 11976.0, |
| "und/clip_ratio/high_mean": 5.278716344037093e-05, |
| "und/clip_ratio/low_mean": 5.7444853155175224e-05, |
| "und/clip_ratio/region_mean": 0.00011023201659554616, |
| "und/kl": 0.17757004161830992, |
| "und/loss": 0.0015241362062852204 |
| }, |
| { |
| "epoch": 2.409927942353883, |
| "grad_norm": 0.031346339732408524, |
| "learning_rate": 1e-05, |
| "loss": 0.0055, |
| "rewards/correctness_reward_func/mean": 0.28125, |
| "rewards/correctness_reward_func/std": 0.7007648944854736, |
| "rewards/strict_format_reward_func/mean": 0.265625, |
| "rewards/strict_format_reward_func/std": 0.2514837086200714, |
| "step": 190, |
| "time_profile/curr_logprobs_and_update": 0.2616291437589098, |
| "time_profile/old_logps": 33.62939000222832, |
| "time_profile/ref_logps": 33.71898835618049, |
| "time_profile/reward": 0.0045094965025782585, |
| "time_profile/text_rollout": 454.1341454349458, |
| "train/gen_step": 12040.0, |
| "und/advantages_abs_mean": 0.437255859375, |
| "und/advantages_max": 2.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.5, |
| "und/advantages_std": 0.6407745480537415, |
| "und/clip_ratio/high_mean": 9.414017040398903e-05, |
| "und/clip_ratio/low_mean": 0.00015810180047992617, |
| "und/clip_ratio/region_mean": 0.0002522419781598728, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.822265625, |
| "und/completion/min_length": 254.0, |
| "und/frac_reward_zero_std": 0.140625, |
| "und/kl": 0.1362051923351828, |
| "und/loss": 0.002659691222561378, |
| "und/prompt/max_length": 49.0, |
| "und/prompt/mean_length": 46.5, |
| "und/prompt/min_length": 44.0, |
| "und/reward": 0.59375, |
| "und/reward_std": 0.846889078617096 |
| }, |
| { |
| "epoch": 2.422738190552442, |
| "grad_norm": 0.026408294215798378, |
| "learning_rate": 1e-05, |
| "loss": 0.0049, |
| "step": 191, |
| "time_profile/curr_logprobs_and_update": 0.2613273377501173, |
| "train/gen_step": 12104.0, |
| "und/clip_ratio/high_mean": 0.00013889491310692392, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.00013889491310692392, |
| "und/kl": 0.12136126437690109, |
| "und/loss": 0.001747753280969988 |
| }, |
| { |
| "epoch": 2.4355484387510007, |
| "grad_norm": 0.03671589121222496, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "rewards/correctness_reward_func/mean": 0.40625, |
| "rewards/correctness_reward_func/std": 0.8110105991363525, |
| "rewards/strict_format_reward_func/mean": 0.34375, |
| "rewards/strict_format_reward_func/std": 0.233588308095932, |
| "step": 192, |
| "time_profile/curr_logprobs_and_update": 0.2613590597611619, |
| "time_profile/old_logps": 33.68849613983184, |
| "time_profile/ref_logps": 33.76824611146003, |
| "time_profile/reward": 0.004611174575984478, |
| "time_profile/text_rollout": 472.31632216088474, |
| "train/gen_step": 12168.0, |
| "und/advantages_abs_mean": 0.55419921875, |
| "und/advantages_max": 1.875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.75, |
| "und/advantages_std": 0.6994109153747559, |
| "und/clip_ratio/high_mean": 7.512019510613754e-05, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 7.512019510613754e-05, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.6640625, |
| "und/completion/min_length": 210.0, |
| "und/frac_reward_zero_std": 0.015625, |
| "und/kl": 0.0662406939372886, |
| "und/loss": 0.0016963802219152058, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 103.0, |
| "und/prompt/min_length": 44.0, |
| "und/reward": 0.8232421875, |
| "und/reward_std": 0.9628657698631287 |
| }, |
| { |
| "epoch": 2.4483586869495597, |
| "grad_norm": 0.04320576414465904, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "step": 193, |
| "time_profile/curr_logprobs_and_update": 0.26153383872588165, |
| "train/gen_step": 12232.0, |
| "und/clip_ratio/high_mean": 5.278716344037093e-05, |
| "und/clip_ratio/low_mean": 4.882812572759576e-05, |
| "und/clip_ratio/region_mean": 0.0001016152891679667, |
| "und/kl": 0.07347345922607929, |
| "und/loss": 0.0033696473110467196 |
| }, |
| { |
| "epoch": 2.4611689351481187, |
| "grad_norm": 0.023478908464312553, |
| "learning_rate": 1e-05, |
| "loss": 0.0034, |
| "rewards/correctness_reward_func/mean": 0.28125, |
| "rewards/correctness_reward_func/std": 0.7007648944854736, |
| "rewards/strict_format_reward_func/mean": 0.3515625, |
| "rewards/strict_format_reward_func/std": 0.2302463799715042, |
| "step": 194, |
| "time_profile/curr_logprobs_and_update": 0.2616785841237288, |
| "time_profile/old_logps": 33.640625360421836, |
| "time_profile/ref_logps": 33.778758310712874, |
| "time_profile/reward": 0.004204129800200462, |
| "time_profile/text_rollout": 472.5064763696864, |
| "train/gen_step": 12296.0, |
| "und/advantages_abs_mean": 0.38671875, |
| "und/advantages_max": 1.8125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.9375, |
| "und/advantages_std": 0.5569894909858704, |
| "und/clip_ratio/high_mean": 0.0001396356092300266, |
| "und/clip_ratio/low_mean": 0.00016529694767086767, |
| "und/clip_ratio/region_mean": 0.00030493255690089427, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.46484375, |
| "und/completion/min_length": 221.0, |
| "und/frac_reward_zero_std": 0.078125, |
| "und/kl": 0.08414672242361121, |
| "und/loss": 0.0038269779179245234, |
| "und/prompt/max_length": 156.0, |
| "und/prompt/mean_length": 107.0, |
| "und/prompt/min_length": 58.0, |
| "und/reward": 0.5478515625, |
| "und/reward_std": 0.6658832430839539 |
| }, |
| { |
| "epoch": 2.4739791833466773, |
| "grad_norm": 0.02270297333598137, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "step": 195, |
| "time_profile/curr_logprobs_and_update": 0.26131977794284467, |
| "train/gen_step": 12360.0, |
| "und/clip_ratio/high_mean": 0.00011789133350248449, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.00011789133350248449, |
| "und/kl": 0.06218441354576498, |
| "und/loss": 0.0017657310236245394 |
| }, |
| { |
| "epoch": 2.4867894315452364, |
| "grad_norm": 0.04379241541028023, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "rewards/correctness_reward_func/mean": 0.5625, |
| "rewards/correctness_reward_func/std": 0.9063270092010498, |
| "rewards/strict_format_reward_func/mean": 0.328125, |
| "rewards/strict_format_reward_func/std": 0.23935678601264954, |
| "step": 196, |
| "time_profile/curr_logprobs_and_update": 0.2615141822316218, |
| "time_profile/old_logps": 33.65821726806462, |
| "time_profile/ref_logps": 33.77625437732786, |
| "time_profile/reward": 0.0044699013233184814, |
| "time_profile/text_rollout": 489.89552684500813, |
| "train/gen_step": 12424.0, |
| "und/advantages_abs_mean": 0.632080078125, |
| "und/advantages_max": 2.125, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.875, |
| "und/advantages_std": 0.8004096150398254, |
| "und/clip_ratio/high_mean": 4.155584974796511e-05, |
| "und/clip_ratio/low_mean": 5.5803571740398183e-05, |
| "und/clip_ratio/region_mean": 9.73594214883633e-05, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.716796875, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.0, |
| "und/kl": 0.05999593347951304, |
| "und/loss": 0.0023087674635462463, |
| "und/prompt/max_length": 162.0, |
| "und/prompt/mean_length": 159.0, |
| "und/prompt/min_length": 156.0, |
| "und/reward": 0.759765625, |
| "und/reward_std": 0.9920932054519653 |
| }, |
| { |
| "epoch": 2.499599679743795, |
| "grad_norm": 0.053688473999500275, |
| "learning_rate": 1e-05, |
| "loss": 0.0033, |
| "step": 197, |
| "time_profile/curr_logprobs_and_update": 0.26162081109941937, |
| "train/gen_step": 12488.0, |
| "und/clip_ratio/high_mean": 0.00011160714348079637, |
| "und/clip_ratio/low_mean": 4.650297705666162e-05, |
| "und/clip_ratio/region_mean": 0.000158110120537458, |
| "und/kl": 0.08334491711866576, |
| "und/loss": 0.002988875610753894 |
| }, |
| { |
| "epoch": 2.512409927942354, |
| "grad_norm": 0.010901146568357944, |
| "learning_rate": 1e-05, |
| "loss": 0.0144, |
| "rewards/correctness_reward_func/mean": 0.84375, |
| "rewards/correctness_reward_func/std": 0.9955257177352905, |
| "rewards/strict_format_reward_func/mean": 0.3515625, |
| "rewards/strict_format_reward_func/std": 0.2302463799715042, |
| "step": 198, |
| "time_profile/curr_logprobs_and_update": 0.2617834777047392, |
| "time_profile/old_logps": 33.705376639030874, |
| "time_profile/ref_logps": 33.77917596697807, |
| "time_profile/reward": 0.004560342989861965, |
| "time_profile/text_rollout": 456.2871036250144, |
| "train/gen_step": 12552.0, |
| "und/advantages_abs_mean": 0.375, |
| "und/advantages_max": 1.625, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -1.9375, |
| "und/advantages_std": 0.5578123331069946, |
| "und/clip_ratio/high_mean": 0.0004084461470483802, |
| "und/clip_ratio/low_mean": 3.9062499126885086e-05, |
| "und/clip_ratio/region_mean": 0.0004475086498132441, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.69921875, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.140625, |
| "und/kl": 0.35732343784184195, |
| "und/loss": 0.002203851705417037, |
| "und/prompt/max_length": 62.0, |
| "und/prompt/mean_length": 54.5, |
| "und/prompt/min_length": 47.0, |
| "und/reward": 1.15234375, |
| "und/reward_std": 1.1305378675460815 |
| }, |
| { |
| "epoch": 2.5252201761409125, |
| "grad_norm": 0.03094113990664482, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "step": 199, |
| "time_profile/curr_logprobs_and_update": 0.2613095892302226, |
| "train/gen_step": 12616.0, |
| "und/clip_ratio/high_mean": 0.0001487574481870979, |
| "und/clip_ratio/low_mean": 0.0, |
| "und/clip_ratio/region_mean": 0.0001487574481870979, |
| "und/kl": 0.05986142602341715, |
| "und/loss": 0.002186899437219836 |
| }, |
| { |
| "epoch": 2.5380304243394716, |
| "grad_norm": 0.03262926638126373, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "rewards/correctness_reward_func/mean": 0.0625, |
| "rewards/correctness_reward_func/std": 0.35073620080947876, |
| "rewards/strict_format_reward_func/mean": 0.0234375, |
| "rewards/strict_format_reward_func/std": 0.10652101784944534, |
| "step": 200, |
| "time_profile/curr_logprobs_and_update": 0.26152220301446505, |
| "time_profile/old_logps": 33.64662089664489, |
| "time_profile/ref_logps": 33.763478820212185, |
| "time_profile/reward": 0.004406346939504147, |
| "time_profile/text_rollout": 519.7042249757797, |
| "train/gen_step": 12680.0, |
| "und/advantages_abs_mean": 0.123779296875, |
| "und/advantages_max": 2.1875, |
| "und/advantages_mean": 0.0, |
| "und/advantages_min": -0.5625, |
| "und/advantages_std": 0.36074504256248474, |
| "und/clip_ratio/high_mean": 5.139802669873461e-05, |
| "und/clip_ratio/low_mean": 5.139802669873461e-05, |
| "und/clip_ratio/region_mean": 0.00010279605339746922, |
| "und/completion/max_length": 255.0, |
| "und/completion/mean_length": 254.958984375, |
| "und/completion/min_length": 253.0, |
| "und/frac_reward_zero_std": 0.640625, |
| "und/kl": 0.05215715864324011, |
| "und/loss": 0.0015407809119096783, |
| "und/prompt/max_length": 247.0, |
| "und/prompt/mean_length": 247.0, |
| "und/prompt/min_length": 247.0, |
| "und/reward": 0.072265625, |
| "und/reward_std": 0.3826628625392914 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 790, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 10, |
| "save_steps": 50, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
