diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,11673 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 50, + "global_step": 3884, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0012873326467559218, + "grad_norm": 652.0, + "learning_rate": 4.99356333676622e-07, + "logits/chosen": -2.5093750953674316, + "logits/rejected": -2.5406250953674316, + "logps/chosen": -287.6000061035156, + "logps/rejected": -303.20001220703125, + "loss": 0.6922, + "rewards/accuracies": 0.2849999964237213, + "rewards/chosen": -0.014697265811264515, + "rewards/margins": 0.008801269344985485, + "rewards/rejected": -0.02348632737994194, + "step": 5 + }, + { + "epoch": 0.0025746652935118436, + "grad_norm": 620.0, + "learning_rate": 4.987126673532441e-07, + "logits/chosen": -2.229687452316284, + "logits/rejected": -2.609375, + "logps/chosen": -290.20001220703125, + "logps/rejected": -283.20001220703125, + "loss": 0.6719, + "rewards/accuracies": 0.4025000035762787, + "rewards/chosen": -0.012036132626235485, + "rewards/margins": 0.05366211012005806, + "rewards/rejected": -0.06558837741613388, + "step": 10 + }, + { + "epoch": 0.0038619979402677654, + "grad_norm": 736.0, + "learning_rate": 4.980690010298661e-07, + "logits/chosen": -2.487499952316284, + "logits/rejected": -2.534374952316284, + "logps/chosen": -312.0, + "logps/rejected": -406.79998779296875, + "loss": 0.6203, + "rewards/accuracies": 0.6680952906608582, + "rewards/chosen": -0.04489745944738388, + "rewards/margins": 0.182861328125, + "rewards/rejected": -0.22783203423023224, + "step": 15 + }, + { + "epoch": 0.005149330587023687, + "grad_norm": 544.0, + "learning_rate": 4.974253347064881e-07, + "logits/chosen": -2.659374952316284, + "logits/rejected": -2.640625, + "logps/chosen": -352.3999938964844, + "logps/rejected": -344.79998779296875, + "loss": 0.6281, + "rewards/accuracies": 0.6508333086967468, + "rewards/chosen": -0.12623290717601776, + "rewards/margins": 0.15234375, + "rewards/rejected": -0.2783203125, + "step": 20 + }, + { + "epoch": 0.006436663233779609, + "grad_norm": 572.0, + "learning_rate": 4.967816683831102e-07, + "logits/chosen": -2.4781250953674316, + "logits/rejected": -2.4124999046325684, + "logps/chosen": -331.20001220703125, + "logps/rejected": -405.6000061035156, + "loss": 0.5758, + "rewards/accuracies": 0.7158333659172058, + "rewards/chosen": -0.24746093153953552, + "rewards/margins": 0.33466798067092896, + "rewards/rejected": -0.581250011920929, + "step": 25 + }, + { + "epoch": 0.007723995880535531, + "grad_norm": 636.0, + "learning_rate": 4.961380020597322e-07, + "logits/chosen": -2.5406250953674316, + "logits/rejected": -2.596874952316284, + "logps/chosen": -307.20001220703125, + "logps/rejected": -341.0, + "loss": 0.5984, + "rewards/accuracies": 0.6928571462631226, + "rewards/chosen": -0.259521484375, + "rewards/margins": 0.29096680879592896, + "rewards/rejected": -0.5497070550918579, + "step": 30 + }, + { + "epoch": 0.009011328527291453, + "grad_norm": 704.0, + "learning_rate": 4.954943357363543e-07, + "logits/chosen": -2.609375, + "logits/rejected": -2.6781249046325684, + "logps/chosen": -252.0, + "logps/rejected": -267.79998779296875, + "loss": 0.6109, + "rewards/accuracies": 0.6726190447807312, + "rewards/chosen": -0.11069335788488388, + "rewards/margins": 0.23808594048023224, + "rewards/rejected": -0.34843748807907104, + "step": 35 + }, + { + "epoch": 0.010298661174047374, + "grad_norm": 652.0, + "learning_rate": 4.948506694129763e-07, + "logits/chosen": -2.674999952316284, + "logits/rejected": -2.7874999046325684, + "logps/chosen": -320.79998779296875, + "logps/rejected": -320.20001220703125, + "loss": 0.6328, + "rewards/accuracies": 0.5697802305221558, + "rewards/chosen": -0.12128295749425888, + "rewards/margins": 0.19589844346046448, + "rewards/rejected": -0.3173828125, + "step": 40 + }, + { + "epoch": 0.011585993820803296, + "grad_norm": 592.0, + "learning_rate": 4.942070030895984e-07, + "logits/chosen": -2.5687499046325684, + "logits/rejected": -2.3968749046325684, + "logps/chosen": -350.6000061035156, + "logps/rejected": -412.3999938964844, + "loss": 0.5512, + "rewards/accuracies": 0.7283333539962769, + "rewards/chosen": -0.3944335877895355, + "rewards/margins": 0.455078125, + "rewards/rejected": -0.848828136920929, + "step": 45 + }, + { + "epoch": 0.012873326467559218, + "grad_norm": 656.0, + "learning_rate": 4.935633367662204e-07, + "logits/chosen": -2.453125, + "logits/rejected": -2.4906249046325684, + "logps/chosen": -306.1000061035156, + "logps/rejected": -337.0, + "loss": 0.5641, + "rewards/accuracies": 0.6176190972328186, + "rewards/chosen": -0.30156248807907104, + "rewards/margins": 0.41289061307907104, + "rewards/rejected": -0.7154296636581421, + "step": 50 + }, + { + "epoch": 0.01416065911431514, + "grad_norm": 812.0, + "learning_rate": 4.929196704428423e-07, + "logits/chosen": -2.637500047683716, + "logits/rejected": -2.340625047683716, + "logps/chosen": -259.79998779296875, + "logps/rejected": -272.70001220703125, + "loss": 0.6133, + "rewards/accuracies": 0.579807698726654, + "rewards/chosen": -0.14125975966453552, + "rewards/margins": 0.2657226622104645, + "rewards/rejected": -0.4073242247104645, + "step": 55 + }, + { + "epoch": 0.015447991761071062, + "grad_norm": 580.0, + "learning_rate": 4.922760041194645e-07, + "logits/chosen": -2.606250047683716, + "logits/rejected": -2.565624952316284, + "logps/chosen": -235.8000030517578, + "logps/rejected": -270.6000061035156, + "loss": 0.6402, + "rewards/accuracies": 0.5483333468437195, + "rewards/chosen": -0.28046876192092896, + "rewards/margins": 0.20820312201976776, + "rewards/rejected": -0.48808592557907104, + "step": 60 + }, + { + "epoch": 0.016735324407826983, + "grad_norm": 636.0, + "learning_rate": 4.916323377960865e-07, + "logits/chosen": -2.489062547683716, + "logits/rejected": -2.535937547683716, + "logps/chosen": -321.6000061035156, + "logps/rejected": -349.6000061035156, + "loss": 0.5641, + "rewards/accuracies": 0.6708333492279053, + "rewards/chosen": -0.371337890625, + "rewards/margins": 0.4146484434604645, + "rewards/rejected": -0.787109375, + "step": 65 + }, + { + "epoch": 0.018022657054582905, + "grad_norm": 588.0, + "learning_rate": 4.909886714727085e-07, + "logits/chosen": -2.5093750953674316, + "logits/rejected": -2.4546875953674316, + "logps/chosen": -325.3999938964844, + "logps/rejected": -326.6000061035156, + "loss": 0.5504, + "rewards/accuracies": 0.6906746625900269, + "rewards/chosen": -0.13496093451976776, + "rewards/margins": 0.42578125, + "rewards/rejected": -0.5609375238418579, + "step": 70 + }, + { + "epoch": 0.019309989701338827, + "grad_norm": 464.0, + "learning_rate": 4.903450051493306e-07, + "logits/chosen": -2.59375, + "logits/rejected": -2.7593750953674316, + "logps/chosen": -337.79998779296875, + "logps/rejected": -289.79998779296875, + "loss": 0.6539, + "rewards/accuracies": 0.5379370450973511, + "rewards/chosen": -0.18120117485523224, + "rewards/margins": 0.18571777641773224, + "rewards/rejected": -0.3662109375, + "step": 75 + }, + { + "epoch": 0.02059732234809475, + "grad_norm": 552.0, + "learning_rate": 4.897013388259526e-07, + "logits/chosen": -2.7093749046325684, + "logits/rejected": -2.643749952316284, + "logps/chosen": -389.6000061035156, + "logps/rejected": -424.3999938964844, + "loss": 0.5195, + "rewards/accuracies": 0.7691667079925537, + "rewards/chosen": -0.06503906100988388, + "rewards/margins": 0.5003906488418579, + "rewards/rejected": -0.565234363079071, + "step": 80 + }, + { + "epoch": 0.02188465499485067, + "grad_norm": 508.0, + "learning_rate": 4.890576725025746e-07, + "logits/chosen": -2.5531249046325684, + "logits/rejected": -2.5406250953674316, + "logps/chosen": -334.0, + "logps/rejected": -432.3999938964844, + "loss": 0.5461, + "rewards/accuracies": 0.6991666555404663, + "rewards/chosen": -0.568591296672821, + "rewards/margins": 0.587695300579071, + "rewards/rejected": -1.157812476158142, + "step": 85 + }, + { + "epoch": 0.023171987641606592, + "grad_norm": 656.0, + "learning_rate": 4.884140061791967e-07, + "logits/chosen": -2.653125047683716, + "logits/rejected": -2.549999952316284, + "logps/chosen": -341.6000061035156, + "logps/rejected": -364.0, + "loss": 0.6277, + "rewards/accuracies": 0.5897619128227234, + "rewards/chosen": -0.4664062559604645, + "rewards/margins": 0.43896484375, + "rewards/rejected": -0.90625, + "step": 90 + }, + { + "epoch": 0.024459320288362514, + "grad_norm": 478.0, + "learning_rate": 4.877703398558187e-07, + "logits/chosen": -2.590625047683716, + "logits/rejected": -2.637500047683716, + "logps/chosen": -340.0, + "logps/rejected": -331.20001220703125, + "loss": 0.5922, + "rewards/accuracies": 0.5897222757339478, + "rewards/chosen": -0.3467773497104645, + "rewards/margins": 0.3914550840854645, + "rewards/rejected": -0.737500011920929, + "step": 95 + }, + { + "epoch": 0.025746652935118436, + "grad_norm": 756.0, + "learning_rate": 4.871266735324407e-07, + "logits/chosen": -2.643749952316284, + "logits/rejected": -2.53125, + "logps/chosen": -379.20001220703125, + "logps/rejected": -404.3999938964844, + "loss": 0.5621, + "rewards/accuracies": 0.6357142925262451, + "rewards/chosen": -0.5185546875, + "rewards/margins": 0.48198240995407104, + "rewards/rejected": -1.0, + "step": 100 + }, + { + "epoch": 0.027033985581874358, + "grad_norm": 784.0, + "learning_rate": 4.864830072090629e-07, + "logits/chosen": -2.5562500953674316, + "logits/rejected": -2.515625, + "logps/chosen": -337.20001220703125, + "logps/rejected": -336.79998779296875, + "loss": 0.6523, + "rewards/accuracies": 0.6341666579246521, + "rewards/chosen": -0.2533203065395355, + "rewards/margins": 0.37348634004592896, + "rewards/rejected": -0.6265624761581421, + "step": 105 + }, + { + "epoch": 0.02832131822863028, + "grad_norm": 470.0, + "learning_rate": 4.858393408856848e-07, + "logits/chosen": -2.4296875, + "logits/rejected": -2.6031250953674316, + "logps/chosen": -329.20001220703125, + "logps/rejected": -373.6000061035156, + "loss": 0.5312, + "rewards/accuracies": 0.6557575464248657, + "rewards/chosen": -0.4478515684604645, + "rewards/margins": 0.53515625, + "rewards/rejected": -0.9837890863418579, + "step": 110 + }, + { + "epoch": 0.0296086508753862, + "grad_norm": 446.0, + "learning_rate": 4.851956745623069e-07, + "logits/chosen": -2.734375, + "logits/rejected": -2.715625047683716, + "logps/chosen": -324.0, + "logps/rejected": -354.3999938964844, + "loss": 0.534, + "rewards/accuracies": 0.7236111164093018, + "rewards/chosen": -0.09278564155101776, + "rewards/margins": 0.5078125, + "rewards/rejected": -0.600390613079071, + "step": 115 + }, + { + "epoch": 0.030895983522142123, + "grad_norm": 492.0, + "learning_rate": 4.845520082389289e-07, + "logits/chosen": -2.653125047683716, + "logits/rejected": -2.5875000953674316, + "logps/chosen": -337.3999938964844, + "logps/rejected": -340.79998779296875, + "loss": 0.568, + "rewards/accuracies": 0.627500057220459, + "rewards/chosen": -0.43541258573532104, + "rewards/margins": 0.37897950410842896, + "rewards/rejected": -0.814013659954071, + "step": 120 + }, + { + "epoch": 0.032183316168898045, + "grad_norm": 600.0, + "learning_rate": 4.839083419155509e-07, + "logits/chosen": -2.78125, + "logits/rejected": -2.5218749046325684, + "logps/chosen": -338.79998779296875, + "logps/rejected": -299.79998779296875, + "loss": 0.6414, + "rewards/accuracies": 0.49666672945022583, + "rewards/chosen": -0.25927734375, + "rewards/margins": 0.20429687201976776, + "rewards/rejected": -0.462890625, + "step": 125 + }, + { + "epoch": 0.03347064881565397, + "grad_norm": 656.0, + "learning_rate": 4.83264675592173e-07, + "logits/chosen": -2.5999999046325684, + "logits/rejected": -2.503124952316284, + "logps/chosen": -295.6000061035156, + "logps/rejected": -354.3999938964844, + "loss": 0.5184, + "rewards/accuracies": 0.7554545402526855, + "rewards/chosen": -0.4603515565395355, + "rewards/margins": 0.614062488079071, + "rewards/rejected": -1.0750000476837158, + "step": 130 + }, + { + "epoch": 0.03475798146240989, + "grad_norm": 476.0, + "learning_rate": 4.82621009268795e-07, + "logits/chosen": -2.635937452316284, + "logits/rejected": -2.5484375953674316, + "logps/chosen": -317.6000061035156, + "logps/rejected": -263.79998779296875, + "loss": 0.6012, + "rewards/accuracies": 0.733055591583252, + "rewards/chosen": -0.19755859673023224, + "rewards/margins": 0.36296385526657104, + "rewards/rejected": -0.560351550579071, + "step": 135 + }, + { + "epoch": 0.03604531410916581, + "grad_norm": 624.0, + "learning_rate": 4.819773429454171e-07, + "logits/chosen": -2.3734374046325684, + "logits/rejected": -2.395312547683716, + "logps/chosen": -326.6000061035156, + "logps/rejected": -309.6000061035156, + "loss": 0.5422, + "rewards/accuracies": 0.7093315720558167, + "rewards/chosen": -0.40888673067092896, + "rewards/margins": 0.5406249761581421, + "rewards/rejected": -0.949999988079071, + "step": 140 + }, + { + "epoch": 0.03733264675592173, + "grad_norm": 660.0, + "learning_rate": 4.813336766220391e-07, + "logits/chosen": -2.518749952316284, + "logits/rejected": -2.637500047683716, + "logps/chosen": -302.6000061035156, + "logps/rejected": -304.45001220703125, + "loss": 0.5453, + "rewards/accuracies": 0.6158334016799927, + "rewards/chosen": -0.376953125, + "rewards/margins": 0.5022948980331421, + "rewards/rejected": -0.880078136920929, + "step": 145 + }, + { + "epoch": 0.038619979402677654, + "grad_norm": 524.0, + "learning_rate": 4.806900102986612e-07, + "logits/chosen": -2.549999952316284, + "logits/rejected": -2.6812500953674316, + "logps/chosen": -318.79998779296875, + "logps/rejected": -322.79998779296875, + "loss": 0.5488, + "rewards/accuracies": 0.6795238256454468, + "rewards/chosen": -0.22490234673023224, + "rewards/margins": 0.501171886920929, + "rewards/rejected": -0.7259765863418579, + "step": 150 + }, + { + "epoch": 0.039907312049433576, + "grad_norm": 556.0, + "learning_rate": 4.800463439752832e-07, + "logits/chosen": -2.4156250953674316, + "logits/rejected": -2.6343750953674316, + "logps/chosen": -194.0, + "logps/rejected": -234.60000610351562, + "loss": 0.5797, + "rewards/accuracies": 0.5741666555404663, + "rewards/chosen": -0.31230467557907104, + "rewards/margins": 0.45820313692092896, + "rewards/rejected": -0.771679699420929, + "step": 155 + }, + { + "epoch": 0.0411946446961895, + "grad_norm": 548.0, + "learning_rate": 4.794026776519052e-07, + "logits/chosen": -2.409374952316284, + "logits/rejected": -2.700000047683716, + "logps/chosen": -209.6999969482422, + "logps/rejected": -219.4499969482422, + "loss": 0.6082, + "rewards/accuracies": 0.6316666603088379, + "rewards/chosen": -0.06511230766773224, + "rewards/margins": 0.3005615174770355, + "rewards/rejected": -0.36503905057907104, + "step": 160 + }, + { + "epoch": 0.04248197734294542, + "grad_norm": 374.0, + "learning_rate": 4.787590113285273e-07, + "logits/chosen": -2.409374952316284, + "logits/rejected": -2.484375, + "logps/chosen": -288.20001220703125, + "logps/rejected": -379.6000061035156, + "loss": 0.4875, + "rewards/accuracies": 0.7725000381469727, + "rewards/chosen": -0.5865234136581421, + "rewards/margins": 0.8304687738418579, + "rewards/rejected": -1.415624976158142, + "step": 165 + }, + { + "epoch": 0.04376930998970134, + "grad_norm": 528.0, + "learning_rate": 4.781153450051493e-07, + "logits/chosen": -2.621875047683716, + "logits/rejected": -2.6624999046325684, + "logps/chosen": -308.3999938964844, + "logps/rejected": -335.20001220703125, + "loss": 0.632, + "rewards/accuracies": 0.5118506550788879, + "rewards/chosen": -0.3763671815395355, + "rewards/margins": 0.3162597715854645, + "rewards/rejected": -0.693554699420929, + "step": 170 + }, + { + "epoch": 0.04505664263645726, + "grad_norm": 884.0, + "learning_rate": 4.774716786817714e-07, + "logits/chosen": -2.307812452316284, + "logits/rejected": -2.40625, + "logps/chosen": -311.0, + "logps/rejected": -350.79998779296875, + "loss": 0.5848, + "rewards/accuracies": 0.6389102935791016, + "rewards/chosen": -0.4908203184604645, + "rewards/margins": 0.4224609434604645, + "rewards/rejected": -0.9117187261581421, + "step": 175 + }, + { + "epoch": 0.046343975283213185, + "grad_norm": 510.0, + "learning_rate": 4.7682801235839336e-07, + "logits/chosen": -2.684375047683716, + "logits/rejected": -2.6078124046325684, + "logps/chosen": -281.0, + "logps/rejected": -296.6000061035156, + "loss": 0.6086, + "rewards/accuracies": 0.6935353875160217, + "rewards/chosen": -0.16635742783546448, + "rewards/margins": 0.36591798067092896, + "rewards/rejected": -0.532421886920929, + "step": 180 + }, + { + "epoch": 0.047631307929969106, + "grad_norm": 532.0, + "learning_rate": 4.7618434603501545e-07, + "logits/chosen": -2.528125047683716, + "logits/rejected": -2.450000047683716, + "logps/chosen": -332.79998779296875, + "logps/rejected": -410.79998779296875, + "loss": 0.498, + "rewards/accuracies": 0.7359615564346313, + "rewards/chosen": -0.4232421815395355, + "rewards/margins": 0.8277343511581421, + "rewards/rejected": -1.2517578601837158, + "step": 185 + }, + { + "epoch": 0.04891864057672503, + "grad_norm": 348.0, + "learning_rate": 4.755406797116375e-07, + "logits/chosen": -2.6875, + "logits/rejected": -2.3734374046325684, + "logps/chosen": -285.0, + "logps/rejected": -398.79998779296875, + "loss": 0.4695, + "rewards/accuracies": 0.7019230723381042, + "rewards/chosen": -0.3487304747104645, + "rewards/margins": 0.783203125, + "rewards/rejected": -1.1328125, + "step": 190 + }, + { + "epoch": 0.05020597322348095, + "grad_norm": 536.0, + "learning_rate": 4.748970133882595e-07, + "logits/chosen": -2.606250047683716, + "logits/rejected": -2.5218749046325684, + "logps/chosen": -285.0, + "logps/rejected": -341.3999938964844, + "loss": 0.4742, + "rewards/accuracies": 0.7864285707473755, + "rewards/chosen": -0.36601561307907104, + "rewards/margins": 0.8246093988418579, + "rewards/rejected": -1.191503882408142, + "step": 195 + }, + { + "epoch": 0.05149330587023687, + "grad_norm": 904.0, + "learning_rate": 4.742533470648816e-07, + "logits/chosen": -2.3843750953674316, + "logits/rejected": -2.534374952316284, + "logps/chosen": -242.60000610351562, + "logps/rejected": -286.79998779296875, + "loss": 0.6508, + "rewards/accuracies": 0.589350700378418, + "rewards/chosen": -0.35893553495407104, + "rewards/margins": 0.37617188692092896, + "rewards/rejected": -0.735156238079071, + "step": 200 + }, + { + "epoch": 0.052780638516992794, + "grad_norm": 508.0, + "learning_rate": 4.7360968074150357e-07, + "logits/chosen": -2.612499952316284, + "logits/rejected": -2.674999952316284, + "logps/chosen": -302.0, + "logps/rejected": -389.3999938964844, + "loss": 0.4621, + "rewards/accuracies": 0.7654762268066406, + "rewards/chosen": -0.4034667909145355, + "rewards/margins": 0.830859363079071, + "rewards/rejected": -1.2355468273162842, + "step": 205 + }, + { + "epoch": 0.054067971163748715, + "grad_norm": 572.0, + "learning_rate": 4.729660144181256e-07, + "logits/chosen": -2.5546875, + "logits/rejected": -2.6031250953674316, + "logps/chosen": -333.0, + "logps/rejected": -372.6000061035156, + "loss": 0.5391, + "rewards/accuracies": 0.6855411529541016, + "rewards/chosen": -0.28925782442092896, + "rewards/margins": 0.63134765625, + "rewards/rejected": -0.919140636920929, + "step": 210 + }, + { + "epoch": 0.05535530381050464, + "grad_norm": 912.0, + "learning_rate": 4.7232234809474765e-07, + "logits/chosen": -2.7093749046325684, + "logits/rejected": -2.390625, + "logps/chosen": -294.79998779296875, + "logps/rejected": -304.3999938964844, + "loss": 0.5695, + "rewards/accuracies": 0.7274725437164307, + "rewards/chosen": -0.10986328125, + "rewards/margins": 0.4535156190395355, + "rewards/rejected": -0.5640624761581421, + "step": 215 + }, + { + "epoch": 0.05664263645726056, + "grad_norm": 484.0, + "learning_rate": 4.716786817713697e-07, + "logits/chosen": -2.6187500953674316, + "logits/rejected": -2.659374952316284, + "logps/chosen": -341.0, + "logps/rejected": -363.29998779296875, + "loss": 0.4887, + "rewards/accuracies": 0.7236905694007874, + "rewards/chosen": -0.4162353575229645, + "rewards/margins": 0.7757812738418579, + "rewards/rejected": -1.1921875476837158, + "step": 220 + }, + { + "epoch": 0.05792996910401648, + "grad_norm": 692.0, + "learning_rate": 4.7103501544799174e-07, + "logits/chosen": -2.784374952316284, + "logits/rejected": -2.799999952316284, + "logps/chosen": -326.20001220703125, + "logps/rejected": -354.0, + "loss": 0.568, + "rewards/accuracies": 0.6629303693771362, + "rewards/chosen": -0.17962035536766052, + "rewards/margins": 0.4164062440395355, + "rewards/rejected": -0.596875011920929, + "step": 225 + }, + { + "epoch": 0.0592173017507724, + "grad_norm": 556.0, + "learning_rate": 4.703913491246138e-07, + "logits/chosen": -2.753124952316284, + "logits/rejected": -2.65625, + "logps/chosen": -334.3999938964844, + "logps/rejected": -329.20001220703125, + "loss": 0.5664, + "rewards/accuracies": 0.6301282644271851, + "rewards/chosen": -0.15971679985523224, + "rewards/margins": 0.48261719942092896, + "rewards/rejected": -0.643359363079071, + "step": 230 + }, + { + "epoch": 0.060504634397528324, + "grad_norm": 712.0, + "learning_rate": 4.697476828012358e-07, + "logits/chosen": -2.7593750953674316, + "logits/rejected": -2.6031250953674316, + "logps/chosen": -344.79998779296875, + "logps/rejected": -366.0, + "loss": 0.5824, + "rewards/accuracies": 0.6634615659713745, + "rewards/chosen": -0.3291015625, + "rewards/margins": 0.4339843690395355, + "rewards/rejected": -0.764453113079071, + "step": 235 + }, + { + "epoch": 0.061791967044284246, + "grad_norm": 604.0, + "learning_rate": 4.6910401647785787e-07, + "logits/chosen": -2.410937547683716, + "logits/rejected": -2.5062499046325684, + "logps/chosen": -263.20001220703125, + "logps/rejected": -294.20001220703125, + "loss": 0.5805, + "rewards/accuracies": 0.6467424631118774, + "rewards/chosen": -0.26542967557907104, + "rewards/margins": 0.4732421934604645, + "rewards/rejected": -0.738085925579071, + "step": 240 + }, + { + "epoch": 0.06307929969104016, + "grad_norm": 370.0, + "learning_rate": 4.6846035015447986e-07, + "logits/chosen": -2.6312499046325684, + "logits/rejected": -2.5999999046325684, + "logps/chosen": -245.0, + "logps/rejected": -295.0, + "loss": 0.5344, + "rewards/accuracies": 0.6723193526268005, + "rewards/chosen": -0.2568359375, + "rewards/margins": 0.5859375, + "rewards/rejected": -0.8433593511581421, + "step": 245 + }, + { + "epoch": 0.06436663233779609, + "grad_norm": 568.0, + "learning_rate": 4.6781668383110195e-07, + "logits/chosen": -2.637500047683716, + "logits/rejected": -2.590625047683716, + "logps/chosen": -295.0, + "logps/rejected": -425.6000061035156, + "loss": 0.4551, + "rewards/accuracies": 0.7516667246818542, + "rewards/chosen": -0.7398437261581421, + "rewards/margins": 1.08984375, + "rewards/rejected": -1.8312499523162842, + "step": 250 + }, + { + "epoch": 0.065653964984552, + "grad_norm": 848.0, + "learning_rate": 4.67173017507724e-07, + "logits/chosen": -2.395312547683716, + "logits/rejected": -2.453125, + "logps/chosen": -331.6000061035156, + "logps/rejected": -402.79998779296875, + "loss": 0.4865, + "rewards/accuracies": 0.7514286041259766, + "rewards/chosen": -0.801562488079071, + "rewards/margins": 0.99609375, + "rewards/rejected": -1.796875, + "step": 255 + }, + { + "epoch": 0.06694129763130793, + "grad_norm": 560.0, + "learning_rate": 4.66529351184346e-07, + "logits/chosen": -2.528125047683716, + "logits/rejected": -2.578125, + "logps/chosen": -346.20001220703125, + "logps/rejected": -408.0, + "loss": 0.4543, + "rewards/accuracies": 0.7574999928474426, + "rewards/chosen": -0.594921886920929, + "rewards/margins": 0.996874988079071, + "rewards/rejected": -1.592187523841858, + "step": 260 + }, + { + "epoch": 0.06822863027806385, + "grad_norm": 600.0, + "learning_rate": 4.658856848609681e-07, + "logits/chosen": -2.4749999046325684, + "logits/rejected": -2.440624952316284, + "logps/chosen": -346.6000061035156, + "logps/rejected": -372.79998779296875, + "loss": 0.5254, + "rewards/accuracies": 0.7100000381469727, + "rewards/chosen": -0.909375011920929, + "rewards/margins": 0.8941406011581421, + "rewards/rejected": -1.8015625476837158, + "step": 265 + }, + { + "epoch": 0.06951596292481978, + "grad_norm": 490.0, + "learning_rate": 4.652420185375901e-07, + "logits/chosen": -2.518749952316284, + "logits/rejected": -2.481250047683716, + "logps/chosen": -384.3999938964844, + "logps/rejected": -431.20001220703125, + "loss": 0.4688, + "rewards/accuracies": 0.746666669845581, + "rewards/chosen": -0.658984363079071, + "rewards/margins": 1.044531226158142, + "rewards/rejected": -1.7023437023162842, + "step": 270 + }, + { + "epoch": 0.07080329557157569, + "grad_norm": 804.0, + "learning_rate": 4.645983522142121e-07, + "logits/chosen": -2.671875, + "logits/rejected": -2.71875, + "logps/chosen": -303.0, + "logps/rejected": -300.79998779296875, + "loss": 0.5121, + "rewards/accuracies": 0.7109615206718445, + "rewards/chosen": -0.4175781309604645, + "rewards/margins": 0.8070312738418579, + "rewards/rejected": -1.2257812023162842, + "step": 275 + }, + { + "epoch": 0.07209062821833162, + "grad_norm": 528.0, + "learning_rate": 4.639546858908342e-07, + "logits/chosen": -2.375, + "logits/rejected": -2.5531249046325684, + "logps/chosen": -358.3999938964844, + "logps/rejected": -348.3999938964844, + "loss": 0.5059, + "rewards/accuracies": 0.7280219793319702, + "rewards/chosen": -0.3921875059604645, + "rewards/margins": 0.6927734613418579, + "rewards/rejected": -1.083593726158142, + "step": 280 + }, + { + "epoch": 0.07337796086508754, + "grad_norm": 676.0, + "learning_rate": 4.633110195674562e-07, + "logits/chosen": -2.364062547683716, + "logits/rejected": -2.2874999046325684, + "logps/chosen": -315.20001220703125, + "logps/rejected": -327.20001220703125, + "loss": 0.675, + "rewards/accuracies": 0.550346314907074, + "rewards/chosen": -0.6898437738418579, + "rewards/margins": 0.4432617127895355, + "rewards/rejected": -1.1355469226837158, + "step": 285 + }, + { + "epoch": 0.07466529351184346, + "grad_norm": 596.0, + "learning_rate": 4.6266735324407824e-07, + "logits/chosen": -2.6343750953674316, + "logits/rejected": -2.609375, + "logps/chosen": -296.3999938964844, + "logps/rejected": -352.6000061035156, + "loss": 0.5039, + "rewards/accuracies": 0.7361721992492676, + "rewards/chosen": -0.5103515386581421, + "rewards/margins": 0.6822265386581421, + "rewards/rejected": -1.19140625, + "step": 290 + }, + { + "epoch": 0.07595262615859938, + "grad_norm": 528.0, + "learning_rate": 4.620236869207003e-07, + "logits/chosen": -2.535937547683716, + "logits/rejected": -2.5, + "logps/chosen": -364.20001220703125, + "logps/rejected": -417.6000061035156, + "loss": 0.4938, + "rewards/accuracies": 0.6839286088943481, + "rewards/chosen": -0.626953125, + "rewards/margins": 0.84765625, + "rewards/rejected": -1.4757812023162842, + "step": 295 + }, + { + "epoch": 0.07723995880535531, + "grad_norm": 752.0, + "learning_rate": 4.613800205973223e-07, + "logits/chosen": -2.671875, + "logits/rejected": -2.6156249046325684, + "logps/chosen": -282.0, + "logps/rejected": -302.79998779296875, + "loss": 0.5855, + "rewards/accuracies": 0.7068431973457336, + "rewards/chosen": -0.3375000059604645, + "rewards/margins": 0.5311523675918579, + "rewards/rejected": -0.868945300579071, + "step": 300 + }, + { + "epoch": 0.07852729145211122, + "grad_norm": 648.0, + "learning_rate": 4.6073635427394437e-07, + "logits/chosen": -2.549999952316284, + "logits/rejected": -2.4375, + "logps/chosen": -358.0, + "logps/rejected": -382.3999938964844, + "loss": 0.5836, + "rewards/accuracies": 0.67083340883255, + "rewards/chosen": -0.7914062738418579, + "rewards/margins": 0.700732409954071, + "rewards/rejected": -1.491796851158142, + "step": 305 + }, + { + "epoch": 0.07981462409886715, + "grad_norm": 652.0, + "learning_rate": 4.6009268795056636e-07, + "logits/chosen": -2.612499952316284, + "logits/rejected": -2.578125, + "logps/chosen": -339.6000061035156, + "logps/rejected": -368.20001220703125, + "loss": 0.4902, + "rewards/accuracies": 0.7871212363243103, + "rewards/chosen": -0.52392578125, + "rewards/margins": 0.864062488079071, + "rewards/rejected": -1.387109398841858, + "step": 310 + }, + { + "epoch": 0.08110195674562307, + "grad_norm": 536.0, + "learning_rate": 4.5944902162718845e-07, + "logits/chosen": -2.6187500953674316, + "logits/rejected": -2.621875047683716, + "logps/chosen": -263.20001220703125, + "logps/rejected": -361.0, + "loss": 0.5008, + "rewards/accuracies": 0.7074023485183716, + "rewards/chosen": -0.38886719942092896, + "rewards/margins": 0.7699218988418579, + "rewards/rejected": -1.157812476158142, + "step": 315 + }, + { + "epoch": 0.082389289392379, + "grad_norm": 460.0, + "learning_rate": 4.588053553038105e-07, + "logits/chosen": -2.715625047683716, + "logits/rejected": -2.7093749046325684, + "logps/chosen": -270.6499938964844, + "logps/rejected": -276.0687561035156, + "loss": 0.6906, + "rewards/accuracies": 0.6304320693016052, + "rewards/chosen": -0.567187488079071, + "rewards/margins": 0.42500001192092896, + "rewards/rejected": -0.992480456829071, + "step": 320 + }, + { + "epoch": 0.08367662203913491, + "grad_norm": 544.0, + "learning_rate": 4.581616889804325e-07, + "logits/chosen": -2.793750047683716, + "logits/rejected": -2.7125000953674316, + "logps/chosen": -319.20001220703125, + "logps/rejected": -325.29998779296875, + "loss": 0.5941, + "rewards/accuracies": 0.6075000166893005, + "rewards/chosen": -0.40937501192092896, + "rewards/margins": 0.4253906309604645, + "rewards/rejected": -0.835156261920929, + "step": 325 + }, + { + "epoch": 0.08496395468589084, + "grad_norm": 608.0, + "learning_rate": 4.575180226570546e-07, + "logits/chosen": -2.559375047683716, + "logits/rejected": -2.4000000953674316, + "logps/chosen": -294.20001220703125, + "logps/rejected": -346.79998779296875, + "loss": 0.4805, + "rewards/accuracies": 0.7804545164108276, + "rewards/chosen": -0.594531238079071, + "rewards/margins": 0.858203113079071, + "rewards/rejected": -1.4523437023162842, + "step": 330 + }, + { + "epoch": 0.08625128733264675, + "grad_norm": 700.0, + "learning_rate": 4.568743563336766e-07, + "logits/chosen": -2.450000047683716, + "logits/rejected": -2.5078125, + "logps/chosen": -275.20001220703125, + "logps/rejected": -256.8999938964844, + "loss": 0.6275, + "rewards/accuracies": 0.6372222304344177, + "rewards/chosen": -0.529101550579071, + "rewards/margins": 0.455810546875, + "rewards/rejected": -0.983203113079071, + "step": 335 + }, + { + "epoch": 0.08753861997940268, + "grad_norm": 328.0, + "learning_rate": 4.562306900102986e-07, + "logits/chosen": -2.6031250953674316, + "logits/rejected": -2.75, + "logps/chosen": -235.5, + "logps/rejected": -257.79998779296875, + "loss": 0.65, + "rewards/accuracies": 0.6473810076713562, + "rewards/chosen": -0.43896484375, + "rewards/margins": 0.3785156309604645, + "rewards/rejected": -0.8163086175918579, + "step": 340 + }, + { + "epoch": 0.0888259526261586, + "grad_norm": 422.0, + "learning_rate": 4.555870236869207e-07, + "logits/chosen": -2.778125047683716, + "logits/rejected": -2.184375047683716, + "logps/chosen": -162.89999389648438, + "logps/rejected": -225.1999969482422, + "loss": 0.5336, + "rewards/accuracies": 0.6917856931686401, + "rewards/chosen": -0.21505126357078552, + "rewards/margins": 0.6953125, + "rewards/rejected": -0.9105468988418579, + "step": 345 + }, + { + "epoch": 0.09011328527291453, + "grad_norm": 382.0, + "learning_rate": 4.549433573635427e-07, + "logits/chosen": -2.565624952316284, + "logits/rejected": -2.59375, + "logps/chosen": -329.20001220703125, + "logps/rejected": -318.20001220703125, + "loss": 0.5301, + "rewards/accuracies": 0.7027472257614136, + "rewards/chosen": -0.2984375059604645, + "rewards/margins": 0.7464843988418579, + "rewards/rejected": -1.0437500476837158, + "step": 350 + }, + { + "epoch": 0.09140061791967044, + "grad_norm": 640.0, + "learning_rate": 4.5429969104016474e-07, + "logits/chosen": -2.5093750953674316, + "logits/rejected": -2.481250047683716, + "logps/chosen": -290.0, + "logps/rejected": -289.0, + "loss": 0.5609, + "rewards/accuracies": 0.6678968071937561, + "rewards/chosen": -0.33183592557907104, + "rewards/margins": 0.5640624761581421, + "rewards/rejected": -0.8960937261581421, + "step": 355 + }, + { + "epoch": 0.09268795056642637, + "grad_norm": 592.0, + "learning_rate": 4.5365602471678684e-07, + "logits/chosen": -2.700000047683716, + "logits/rejected": -2.606250047683716, + "logps/chosen": -320.3999938964844, + "logps/rejected": -354.3999938964844, + "loss": 0.5668, + "rewards/accuracies": 0.6139103174209595, + "rewards/chosen": -0.38642579317092896, + "rewards/margins": 0.6519531011581421, + "rewards/rejected": -1.038671851158142, + "step": 360 + }, + { + "epoch": 0.09397528321318228, + "grad_norm": 462.0, + "learning_rate": 4.5301235839340883e-07, + "logits/chosen": -2.5250000953674316, + "logits/rejected": -2.621875047683716, + "logps/chosen": -355.6000061035156, + "logps/rejected": -346.20001220703125, + "loss": 0.5492, + "rewards/accuracies": 0.636783242225647, + "rewards/chosen": -0.5849609375, + "rewards/margins": 0.805859386920929, + "rewards/rejected": -1.392187476158142, + "step": 365 + }, + { + "epoch": 0.09526261585993821, + "grad_norm": 540.0, + "learning_rate": 4.5236869207003087e-07, + "logits/chosen": -2.543750047683716, + "logits/rejected": -2.4625000953674316, + "logps/chosen": -312.0, + "logps/rejected": -344.79998779296875, + "loss": 0.5098, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.42585450410842896, + "rewards/margins": 0.801953136920929, + "rewards/rejected": -1.228124976158142, + "step": 370 + }, + { + "epoch": 0.09654994850669413, + "grad_norm": 676.0, + "learning_rate": 4.517250257466529e-07, + "logits/chosen": -2.59375, + "logits/rejected": -2.6312499046325684, + "logps/chosen": -360.79998779296875, + "logps/rejected": -383.6000061035156, + "loss": 0.4746, + "rewards/accuracies": 0.7425000071525574, + "rewards/chosen": -0.4580078125, + "rewards/margins": 0.8765624761581421, + "rewards/rejected": -1.3359375, + "step": 375 + }, + { + "epoch": 0.09783728115345006, + "grad_norm": 700.0, + "learning_rate": 4.5108135942327496e-07, + "logits/chosen": -2.731250047683716, + "logits/rejected": -2.746875047683716, + "logps/chosen": -317.20001220703125, + "logps/rejected": -280.79998779296875, + "loss": 0.643, + "rewards/accuracies": 0.5761904716491699, + "rewards/chosen": -0.49604493379592896, + "rewards/margins": 0.41357421875, + "rewards/rejected": -0.907031238079071, + "step": 380 + }, + { + "epoch": 0.09912461380020597, + "grad_norm": 416.0, + "learning_rate": 4.50437693099897e-07, + "logits/chosen": -2.6500000953674316, + "logits/rejected": -2.799999952316284, + "logps/chosen": -325.79998779296875, + "logps/rejected": -280.125, + "loss": 0.5805, + "rewards/accuracies": 0.6991666555404663, + "rewards/chosen": -0.2798828184604645, + "rewards/margins": 0.4990234375, + "rewards/rejected": -0.778393566608429, + "step": 385 + }, + { + "epoch": 0.1004119464469619, + "grad_norm": 720.0, + "learning_rate": 4.49794026776519e-07, + "logits/chosen": -2.590625047683716, + "logits/rejected": -2.614062547683716, + "logps/chosen": -353.3999938964844, + "logps/rejected": -360.0, + "loss": 0.525, + "rewards/accuracies": 0.659166693687439, + "rewards/chosen": -0.5311523675918579, + "rewards/margins": 0.636523425579071, + "rewards/rejected": -1.1671874523162842, + "step": 390 + }, + { + "epoch": 0.10169927909371781, + "grad_norm": 1152.0, + "learning_rate": 4.491503604531411e-07, + "logits/chosen": -2.4671874046325684, + "logits/rejected": -2.403125047683716, + "logps/chosen": -304.3999938964844, + "logps/rejected": -258.79998779296875, + "loss": 0.6543, + "rewards/accuracies": 0.5882143378257751, + "rewards/chosen": -0.518359363079071, + "rewards/margins": 0.4415039122104645, + "rewards/rejected": -0.960156261920929, + "step": 395 + }, + { + "epoch": 0.10298661174047374, + "grad_norm": 462.0, + "learning_rate": 4.4850669412976313e-07, + "logits/chosen": -2.359375, + "logits/rejected": -2.375, + "logps/chosen": -263.79998779296875, + "logps/rejected": -323.79998779296875, + "loss": 0.493, + "rewards/accuracies": 0.7314102649688721, + "rewards/chosen": -0.24116210639476776, + "rewards/margins": 0.748046875, + "rewards/rejected": -0.987500011920929, + "step": 400 + }, + { + "epoch": 0.10427394438722966, + "grad_norm": 354.0, + "learning_rate": 4.478630278063851e-07, + "logits/chosen": -2.559375047683716, + "logits/rejected": -2.6500000953674316, + "logps/chosen": -296.3999938964844, + "logps/rejected": -364.0, + "loss": 0.4609, + "rewards/accuracies": 0.7352855801582336, + "rewards/chosen": -0.4027343690395355, + "rewards/margins": 0.8873046636581421, + "rewards/rejected": -1.2902343273162842, + "step": 405 + }, + { + "epoch": 0.10556127703398559, + "grad_norm": 696.0, + "learning_rate": 4.472193614830072e-07, + "logits/chosen": -2.596874952316284, + "logits/rejected": -2.7593750953674316, + "logps/chosen": -314.79998779296875, + "logps/rejected": -334.6000061035156, + "loss": 0.5, + "rewards/accuracies": 0.7326923608779907, + "rewards/chosen": -0.4986816346645355, + "rewards/margins": 0.758984386920929, + "rewards/rejected": -1.2570312023162842, + "step": 410 + }, + { + "epoch": 0.1068486096807415, + "grad_norm": 472.0, + "learning_rate": 4.4657569515962926e-07, + "logits/chosen": -2.4671874046325684, + "logits/rejected": -2.6468749046325684, + "logps/chosen": -249.60000610351562, + "logps/rejected": -257.79998779296875, + "loss": 0.6188, + "rewards/accuracies": 0.5436111688613892, + "rewards/chosen": -0.2529296875, + "rewards/margins": 0.46904295682907104, + "rewards/rejected": -0.7203124761581421, + "step": 415 + }, + { + "epoch": 0.10813594232749743, + "grad_norm": 480.0, + "learning_rate": 4.4593202883625124e-07, + "logits/chosen": -2.4828124046325684, + "logits/rejected": -2.628124952316284, + "logps/chosen": -315.0, + "logps/rejected": -368.0, + "loss": 0.5004, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.5386718511581421, + "rewards/margins": 0.832812488079071, + "rewards/rejected": -1.37109375, + "step": 420 + }, + { + "epoch": 0.10942327497425335, + "grad_norm": 482.0, + "learning_rate": 4.4528836251287334e-07, + "logits/chosen": -2.6812500953674316, + "logits/rejected": -2.674999952316284, + "logps/chosen": -296.5, + "logps/rejected": -326.8999938964844, + "loss": 0.6176, + "rewards/accuracies": 0.6120238304138184, + "rewards/chosen": -0.379150390625, + "rewards/margins": 0.4447265565395355, + "rewards/rejected": -0.824023425579071, + "step": 425 + }, + { + "epoch": 0.11071060762100927, + "grad_norm": 410.0, + "learning_rate": 4.4464469618949533e-07, + "logits/chosen": -2.534374952316284, + "logits/rejected": -2.4625000953674316, + "logps/chosen": -276.6000061035156, + "logps/rejected": -343.79998779296875, + "loss": 0.5086, + "rewards/accuracies": 0.6891666650772095, + "rewards/chosen": -0.441162109375, + "rewards/margins": 0.8179687261581421, + "rewards/rejected": -1.259374976158142, + "step": 430 + }, + { + "epoch": 0.11199794026776519, + "grad_norm": 544.0, + "learning_rate": 4.4400102986611737e-07, + "logits/chosen": -2.6968750953674316, + "logits/rejected": -2.703125, + "logps/chosen": -355.6000061035156, + "logps/rejected": -395.20001220703125, + "loss": 0.5211, + "rewards/accuracies": 0.6591666340827942, + "rewards/chosen": -0.46074217557907104, + "rewards/margins": 0.727343738079071, + "rewards/rejected": -1.189062476158142, + "step": 435 + }, + { + "epoch": 0.11328527291452112, + "grad_norm": 494.0, + "learning_rate": 4.4335736354273947e-07, + "logits/chosen": -2.59375, + "logits/rejected": -2.6343750953674316, + "logps/chosen": -342.3999938964844, + "logps/rejected": -346.3999938964844, + "loss": 0.623, + "rewards/accuracies": 0.588809609413147, + "rewards/chosen": -0.41386717557907104, + "rewards/margins": 0.45361328125, + "rewards/rejected": -0.8667968511581421, + "step": 440 + }, + { + "epoch": 0.11457260556127703, + "grad_norm": 330.0, + "learning_rate": 4.4271369721936146e-07, + "logits/chosen": -2.3499999046325684, + "logits/rejected": -2.34375, + "logps/chosen": -303.0, + "logps/rejected": -368.6000061035156, + "loss": 0.4627, + "rewards/accuracies": 0.7829545736312866, + "rewards/chosen": -0.784375011920929, + "rewards/margins": 0.9273437261581421, + "rewards/rejected": -1.709375023841858, + "step": 445 + }, + { + "epoch": 0.11585993820803296, + "grad_norm": 812.0, + "learning_rate": 4.420700308959835e-07, + "logits/chosen": -2.643749952316284, + "logits/rejected": -2.471874952316284, + "logps/chosen": -319.20001220703125, + "logps/rejected": -401.6000061035156, + "loss": 0.5406, + "rewards/accuracies": 0.7620238065719604, + "rewards/chosen": -0.4869140684604645, + "rewards/margins": 0.8675781488418579, + "rewards/rejected": -1.3562500476837158, + "step": 450 + }, + { + "epoch": 0.11714727085478888, + "grad_norm": 820.0, + "learning_rate": 4.414263645726055e-07, + "logits/chosen": -2.5374999046325684, + "logits/rejected": -2.5625, + "logps/chosen": -345.20001220703125, + "logps/rejected": -428.0, + "loss": 0.4703, + "rewards/accuracies": 0.7591666579246521, + "rewards/chosen": -1.0148437023162842, + "rewards/margins": 1.153906226158142, + "rewards/rejected": -2.1656250953674316, + "step": 455 + }, + { + "epoch": 0.1184346035015448, + "grad_norm": 800.0, + "learning_rate": 4.407826982492276e-07, + "logits/chosen": -2.515625, + "logits/rejected": -2.4749999046325684, + "logps/chosen": -303.0, + "logps/rejected": -352.0, + "loss": 0.4973, + "rewards/accuracies": 0.7733974456787109, + "rewards/chosen": -0.673632800579071, + "rewards/margins": 0.82763671875, + "rewards/rejected": -1.501562476158142, + "step": 460 + }, + { + "epoch": 0.11972193614830072, + "grad_norm": 368.0, + "learning_rate": 4.4013903192584963e-07, + "logits/chosen": -2.6624999046325684, + "logits/rejected": -2.6171875, + "logps/chosen": -256.5, + "logps/rejected": -322.20001220703125, + "loss": 0.4992, + "rewards/accuracies": 0.6505128145217896, + "rewards/chosen": -0.19936522841453552, + "rewards/margins": 0.6820312738418579, + "rewards/rejected": -0.8828125, + "step": 465 + }, + { + "epoch": 0.12100926879505665, + "grad_norm": 628.0, + "learning_rate": 4.394953656024716e-07, + "logits/chosen": -2.640625, + "logits/rejected": -2.5625, + "logps/chosen": -325.6000061035156, + "logps/rejected": -331.0, + "loss": 0.5191, + "rewards/accuracies": 0.7626281976699829, + "rewards/chosen": -0.41020506620407104, + "rewards/margins": 0.7154296636581421, + "rewards/rejected": -1.126562476158142, + "step": 470 + }, + { + "epoch": 0.12229660144181256, + "grad_norm": 768.0, + "learning_rate": 4.388516992790937e-07, + "logits/chosen": -2.549999952316284, + "logits/rejected": -2.4937500953674316, + "logps/chosen": -307.6000061035156, + "logps/rejected": -406.0, + "loss": 0.4379, + "rewards/accuracies": 0.7322435975074768, + "rewards/chosen": -0.6890624761581421, + "rewards/margins": 1.10546875, + "rewards/rejected": -1.795312523841858, + "step": 475 + }, + { + "epoch": 0.12358393408856849, + "grad_norm": 948.0, + "learning_rate": 4.3820803295571576e-07, + "logits/chosen": -2.4593749046325684, + "logits/rejected": -2.5625, + "logps/chosen": -313.0, + "logps/rejected": -323.45001220703125, + "loss": 0.5887, + "rewards/accuracies": 0.6767857074737549, + "rewards/chosen": -0.3560546934604645, + "rewards/margins": 0.5638672113418579, + "rewards/rejected": -0.92236328125, + "step": 480 + }, + { + "epoch": 0.12487126673532441, + "grad_norm": 568.0, + "learning_rate": 4.3756436663233775e-07, + "logits/chosen": NaN, + "logits/rejected": -2.671875, + "logps/chosen": -284.0, + "logps/rejected": -336.23126220703125, + "loss": 0.5758, + "rewards/accuracies": 0.5916666984558105, + "rewards/chosen": -0.4429687559604645, + "rewards/margins": 0.720703125, + "rewards/rejected": -1.1642577648162842, + "step": 485 + }, + { + "epoch": 0.12615859938208032, + "grad_norm": 394.0, + "learning_rate": 4.3692070030895984e-07, + "logits/chosen": -2.5218749046325684, + "logits/rejected": -2.5250000953674316, + "logps/chosen": -268.0, + "logps/rejected": -427.6000061035156, + "loss": 0.4879, + "rewards/accuracies": 0.7219139337539673, + "rewards/chosen": -0.7378906011581421, + "rewards/margins": 1.002343773841858, + "rewards/rejected": -1.736718773841858, + "step": 490 + }, + { + "epoch": 0.12744593202883625, + "grad_norm": 402.0, + "learning_rate": 4.3627703398558183e-07, + "logits/chosen": -2.4437499046325684, + "logits/rejected": -2.4000000953674316, + "logps/chosen": -276.6000061035156, + "logps/rejected": -374.0, + "loss": 0.4957, + "rewards/accuracies": 0.659166693687439, + "rewards/chosen": -0.3246093690395355, + "rewards/margins": 0.7515624761581421, + "rewards/rejected": -1.074609398841858, + "step": 495 + }, + { + "epoch": 0.12873326467559218, + "grad_norm": 378.0, + "learning_rate": 4.356333676622039e-07, + "logits/chosen": -2.5093750953674316, + "logits/rejected": -2.2890625, + "logps/chosen": -276.20001220703125, + "logps/rejected": -299.70001220703125, + "loss": 0.525, + "rewards/accuracies": 0.7402380704879761, + "rewards/chosen": -0.3895507752895355, + "rewards/margins": 0.840624988079071, + "rewards/rejected": -1.230078101158142, + "step": 500 + }, + { + "epoch": 0.1300205973223481, + "grad_norm": 454.0, + "learning_rate": 4.3498970133882597e-07, + "logits/chosen": NaN, + "logits/rejected": -2.5406250953674316, + "logps/chosen": -251.60000610351562, + "logps/rejected": -261.11248779296875, + "loss": 0.5547, + "rewards/accuracies": 0.7416666746139526, + "rewards/chosen": -0.44453126192092896, + "rewards/margins": 0.686767578125, + "rewards/rejected": -1.129150390625, + "step": 505 + }, + { + "epoch": 0.131307929969104, + "grad_norm": 310.0, + "learning_rate": 4.3434603501544796e-07, + "logits/chosen": -2.6031250953674316, + "logits/rejected": -2.640625, + "logps/chosen": -317.3999938964844, + "logps/rejected": -376.79998779296875, + "loss": 0.4156, + "rewards/accuracies": 0.8378571271896362, + "rewards/chosen": -0.644726574420929, + "rewards/margins": 1.0974609851837158, + "rewards/rejected": -1.7421875, + "step": 510 + }, + { + "epoch": 0.13259526261585994, + "grad_norm": 504.0, + "learning_rate": 4.3370236869207e-07, + "logits/chosen": -2.434375047683716, + "logits/rejected": -2.581249952316284, + "logps/chosen": -246.0, + "logps/rejected": -273.6000061035156, + "loss": 0.5074, + "rewards/accuracies": 0.730335533618927, + "rewards/chosen": -0.40351563692092896, + "rewards/margins": 0.762890636920929, + "rewards/rejected": -1.1648437976837158, + "step": 515 + }, + { + "epoch": 0.13388259526261587, + "grad_norm": 348.0, + "learning_rate": 4.330587023686921e-07, + "logits/chosen": -2.559375047683716, + "logits/rejected": -2.6156249046325684, + "logps/chosen": -309.0, + "logps/rejected": -384.3999938964844, + "loss": 0.5305, + "rewards/accuracies": 0.7691666483879089, + "rewards/chosen": -0.782031238079071, + "rewards/margins": 0.828125, + "rewards/rejected": -1.607812523841858, + "step": 520 + }, + { + "epoch": 0.1351699279093718, + "grad_norm": 360.0, + "learning_rate": 4.324150360453141e-07, + "logits/chosen": -2.815624952316284, + "logits/rejected": -2.8968749046325684, + "logps/chosen": -342.79998779296875, + "logps/rejected": -274.0, + "loss": 0.6109, + "rewards/accuracies": 0.6179870367050171, + "rewards/chosen": -0.2759765684604645, + "rewards/margins": 0.33916014432907104, + "rewards/rejected": -0.6146484613418579, + "step": 525 + }, + { + "epoch": 0.1364572605561277, + "grad_norm": 744.0, + "learning_rate": 4.3177136972193613e-07, + "logits/chosen": -2.6015625, + "logits/rejected": -2.5625, + "logps/chosen": -290.79998779296875, + "logps/rejected": -385.6000061035156, + "loss": 0.4789, + "rewards/accuracies": 0.7050000429153442, + "rewards/chosen": -0.9300781488418579, + "rewards/margins": 1.2216796875, + "rewards/rejected": -2.1500000953674316, + "step": 530 + }, + { + "epoch": 0.13774459320288363, + "grad_norm": 560.0, + "learning_rate": 4.311277033985581e-07, + "logits/chosen": -2.6624999046325684, + "logits/rejected": -2.6343750953674316, + "logps/chosen": -280.3999938964844, + "logps/rejected": -359.6000061035156, + "loss": 0.5152, + "rewards/accuracies": 0.7096611857414246, + "rewards/chosen": -0.4908203184604645, + "rewards/margins": 0.731249988079071, + "rewards/rejected": -1.220312476158142, + "step": 535 + }, + { + "epoch": 0.13903192584963955, + "grad_norm": 544.0, + "learning_rate": 4.304840370751802e-07, + "logits/chosen": -2.5531249046325684, + "logits/rejected": -2.249218702316284, + "logps/chosen": -298.79998779296875, + "logps/rejected": -350.20001220703125, + "loss": 0.5613, + "rewards/accuracies": 0.6763461828231812, + "rewards/chosen": -0.33820801973342896, + "rewards/margins": 0.653076171875, + "rewards/rejected": -0.990771472454071, + "step": 540 + }, + { + "epoch": 0.14031925849639545, + "grad_norm": 338.0, + "learning_rate": 4.2984037075180226e-07, + "logits/chosen": -2.440624952316284, + "logits/rejected": -2.543750047683716, + "logps/chosen": -285.1000061035156, + "logps/rejected": -326.79998779296875, + "loss": 0.5012, + "rewards/accuracies": 0.656847357749939, + "rewards/chosen": -0.531445324420929, + "rewards/margins": 0.865039050579071, + "rewards/rejected": -1.396875023841858, + "step": 545 + }, + { + "epoch": 0.14160659114315138, + "grad_norm": 580.0, + "learning_rate": 4.2919670442842425e-07, + "logits/chosen": -2.6875, + "logits/rejected": -2.734375, + "logps/chosen": -332.6000061035156, + "logps/rejected": -362.0, + "loss": 0.5051, + "rewards/accuracies": 0.7197222113609314, + "rewards/chosen": -0.7406250238418579, + "rewards/margins": 0.9212890863418579, + "rewards/rejected": -1.664453148841858, + "step": 550 + }, + { + "epoch": 0.1428939237899073, + "grad_norm": 864.0, + "learning_rate": 4.2855303810504634e-07, + "logits/chosen": -2.643749952316284, + "logits/rejected": -2.6781249046325684, + "logps/chosen": -267.20001220703125, + "logps/rejected": -295.3999938964844, + "loss": 0.5762, + "rewards/accuracies": 0.6717948913574219, + "rewards/chosen": -0.5941406488418579, + "rewards/margins": 0.4912109375, + "rewards/rejected": -1.0859375, + "step": 555 + }, + { + "epoch": 0.14418125643666324, + "grad_norm": 378.0, + "learning_rate": 4.2790937178166833e-07, + "logits/chosen": -2.6343750953674316, + "logits/rejected": -2.5625, + "logps/chosen": -265.6000061035156, + "logps/rejected": -304.6000061035156, + "loss": 0.5062, + "rewards/accuracies": 0.6267856955528259, + "rewards/chosen": -0.3727783262729645, + "rewards/margins": 0.79833984375, + "rewards/rejected": -1.172949194908142, + "step": 560 + }, + { + "epoch": 0.14546858908341914, + "grad_norm": 364.0, + "learning_rate": 4.272657054582904e-07, + "logits/chosen": -2.559375047683716, + "logits/rejected": -2.3046875, + "logps/chosen": -315.0, + "logps/rejected": -332.20001220703125, + "loss": 0.577, + "rewards/accuracies": 0.6833333373069763, + "rewards/chosen": -0.708984375, + "rewards/margins": 0.6767822504043579, + "rewards/rejected": -1.3835937976837158, + "step": 565 + }, + { + "epoch": 0.14675592173017507, + "grad_norm": 488.0, + "learning_rate": 4.2662203913491247e-07, + "logits/chosen": -2.5843749046325684, + "logits/rejected": -2.270312547683716, + "logps/chosen": -289.3999938964844, + "logps/rejected": -325.6000061035156, + "loss": 0.5445, + "rewards/accuracies": 0.7154095768928528, + "rewards/chosen": -0.5423828363418579, + "rewards/margins": 0.831250011920929, + "rewards/rejected": -1.372656226158142, + "step": 570 + }, + { + "epoch": 0.148043254376931, + "grad_norm": 576.0, + "learning_rate": 4.2597837281153446e-07, + "logits/chosen": -2.534374952316284, + "logits/rejected": -2.528125047683716, + "logps/chosen": -254.60000610351562, + "logps/rejected": -379.3999938964844, + "loss": 0.5102, + "rewards/accuracies": 0.6790842413902283, + "rewards/chosen": -0.64453125, + "rewards/margins": 0.969531238079071, + "rewards/rejected": -1.6124999523162842, + "step": 575 + }, + { + "epoch": 0.14933058702368693, + "grad_norm": 390.0, + "learning_rate": 4.253347064881565e-07, + "logits/chosen": -2.512500047683716, + "logits/rejected": -2.2093749046325684, + "logps/chosen": -310.79998779296875, + "logps/rejected": -367.3999938964844, + "loss": 0.4668, + "rewards/accuracies": 0.7688491940498352, + "rewards/chosen": -0.7745116949081421, + "rewards/margins": 0.997265636920929, + "rewards/rejected": -1.7703125476837158, + "step": 580 + }, + { + "epoch": 0.15061791967044283, + "grad_norm": 596.0, + "learning_rate": 4.246910401647786e-07, + "logits/chosen": -2.6343750953674316, + "logits/rejected": -2.621875047683716, + "logps/chosen": -333.6000061035156, + "logps/rejected": -349.79998779296875, + "loss": 0.5133, + "rewards/accuracies": 0.7504370808601379, + "rewards/chosen": -0.567187488079071, + "rewards/margins": 0.813281238079071, + "rewards/rejected": -1.3839843273162842, + "step": 585 + }, + { + "epoch": 0.15190525231719876, + "grad_norm": 424.0, + "learning_rate": 4.240473738414006e-07, + "logits/chosen": -2.7718749046325684, + "logits/rejected": -2.6656250953674316, + "logps/chosen": -297.0, + "logps/rejected": -266.5, + "loss": 0.643, + "rewards/accuracies": 0.5136796832084656, + "rewards/chosen": -0.33324965834617615, + "rewards/margins": 0.29975587129592896, + "rewards/rejected": -0.6332031488418579, + "step": 590 + }, + { + "epoch": 0.1531925849639547, + "grad_norm": 400.0, + "learning_rate": 4.2340370751802263e-07, + "logits/chosen": -2.5531249046325684, + "logits/rejected": -2.653125047683716, + "logps/chosen": -344.79998779296875, + "logps/rejected": -355.20001220703125, + "loss": 0.4445, + "rewards/accuracies": 0.748106062412262, + "rewards/chosen": -0.46464842557907104, + "rewards/margins": 1.052734375, + "rewards/rejected": -1.5183594226837158, + "step": 595 + }, + { + "epoch": 0.15447991761071062, + "grad_norm": 532.0, + "learning_rate": 4.227600411946447e-07, + "logits/chosen": -2.59375, + "logits/rejected": -2.5062499046325684, + "logps/chosen": -301.3999938964844, + "logps/rejected": -316.98748779296875, + "loss": 0.4648, + "rewards/accuracies": 0.7803571820259094, + "rewards/chosen": -0.680468738079071, + "rewards/margins": 0.9820312261581421, + "rewards/rejected": -1.665624976158142, + "step": 600 + }, + { + "epoch": 0.15576725025746652, + "grad_norm": 536.0, + "learning_rate": 4.221163748712667e-07, + "logits/chosen": -2.581249952316284, + "logits/rejected": -2.6156249046325684, + "logps/chosen": -399.20001220703125, + "logps/rejected": -404.79998779296875, + "loss": 0.5156, + "rewards/accuracies": 0.7620879411697388, + "rewards/chosen": -0.698437511920929, + "rewards/margins": 0.835156261920929, + "rewards/rejected": -1.5343749523162842, + "step": 605 + }, + { + "epoch": 0.15705458290422245, + "grad_norm": 960.0, + "learning_rate": 4.2147270854788876e-07, + "logits/chosen": -2.6468749046325684, + "logits/rejected": -2.504687547683716, + "logps/chosen": -278.1000061035156, + "logps/rejected": -318.70001220703125, + "loss": 0.6113, + "rewards/accuracies": 0.6079762578010559, + "rewards/chosen": -0.7206054925918579, + "rewards/margins": 0.734570324420929, + "rewards/rejected": -1.4572265148162842, + "step": 610 + }, + { + "epoch": 0.15834191555097837, + "grad_norm": 696.0, + "learning_rate": 4.208290422245108e-07, + "logits/chosen": -2.6343750953674316, + "logits/rejected": -2.6968750953674316, + "logps/chosen": -305.8999938964844, + "logps/rejected": -334.3999938964844, + "loss": 0.5762, + "rewards/accuracies": 0.5940584540367126, + "rewards/chosen": -0.5921630859375, + "rewards/margins": 0.9027343988418579, + "rewards/rejected": -1.493749976158142, + "step": 615 + }, + { + "epoch": 0.1596292481977343, + "grad_norm": 628.0, + "learning_rate": 4.2018537590113285e-07, + "logits/chosen": -2.643749952316284, + "logits/rejected": -2.671875, + "logps/chosen": -290.3999938964844, + "logps/rejected": -410.79998779296875, + "loss": 0.5078, + "rewards/accuracies": 0.6861904859542847, + "rewards/chosen": -0.7027343511581421, + "rewards/margins": 0.966015636920929, + "rewards/rejected": -1.66796875, + "step": 620 + }, + { + "epoch": 0.1609165808444902, + "grad_norm": 370.0, + "learning_rate": 4.195417095777549e-07, + "logits/chosen": -2.4937500953674316, + "logits/rejected": -2.481250047683716, + "logps/chosen": -271.0, + "logps/rejected": -313.3999938964844, + "loss": 0.5141, + "rewards/accuracies": 0.7365476489067078, + "rewards/chosen": -0.4833984375, + "rewards/margins": 0.8753906488418579, + "rewards/rejected": -1.357812523841858, + "step": 625 + }, + { + "epoch": 0.16220391349124613, + "grad_norm": 392.0, + "learning_rate": 4.188980432543769e-07, + "logits/chosen": -2.621875047683716, + "logits/rejected": -2.4312500953674316, + "logps/chosen": -267.20001220703125, + "logps/rejected": -277.3999938964844, + "loss": 0.4516, + "rewards/accuracies": 0.6999242901802063, + "rewards/chosen": -0.5703125, + "rewards/margins": 1.141015648841858, + "rewards/rejected": -1.712499976158142, + "step": 630 + }, + { + "epoch": 0.16349124613800206, + "grad_norm": 784.0, + "learning_rate": 4.18254376930999e-07, + "logits/chosen": -2.518749952316284, + "logits/rejected": -2.4203124046325684, + "logps/chosen": -328.20001220703125, + "logps/rejected": -371.3999938964844, + "loss": 0.4613, + "rewards/accuracies": 0.7785714268684387, + "rewards/chosen": -0.630664050579071, + "rewards/margins": 0.930859386920929, + "rewards/rejected": -1.563867211341858, + "step": 635 + }, + { + "epoch": 0.164778578784758, + "grad_norm": 404.0, + "learning_rate": 4.1761071060762096e-07, + "logits/chosen": -2.700000047683716, + "logits/rejected": -2.684375047683716, + "logps/chosen": -286.3999938964844, + "logps/rejected": -324.3999938964844, + "loss": 0.5023, + "rewards/accuracies": 0.7629762291908264, + "rewards/chosen": -0.48457032442092896, + "rewards/margins": 0.814648449420929, + "rewards/rejected": -1.299218773841858, + "step": 640 + }, + { + "epoch": 0.1660659114315139, + "grad_norm": 636.0, + "learning_rate": 4.16967044284243e-07, + "logits/chosen": -2.621875047683716, + "logits/rejected": -2.6875, + "logps/chosen": -331.6000061035156, + "logps/rejected": -355.20001220703125, + "loss": 0.5242, + "rewards/accuracies": 0.7101190686225891, + "rewards/chosen": -0.682421863079071, + "rewards/margins": 0.621289074420929, + "rewards/rejected": -1.302343726158142, + "step": 645 + }, + { + "epoch": 0.16735324407826982, + "grad_norm": 478.0, + "learning_rate": 4.163233779608651e-07, + "logits/chosen": -2.643749952316284, + "logits/rejected": -2.5625, + "logps/chosen": -290.6000061035156, + "logps/rejected": -280.0, + "loss": 0.5051, + "rewards/accuracies": 0.6437118649482727, + "rewards/chosen": -0.32392579317092896, + "rewards/margins": 0.6818603277206421, + "rewards/rejected": -1.005468726158142, + "step": 650 + }, + { + "epoch": 0.16864057672502575, + "grad_norm": 596.0, + "learning_rate": 4.156797116374871e-07, + "logits/chosen": -2.426562547683716, + "logits/rejected": -2.371875047683716, + "logps/chosen": -192.64999389648438, + "logps/rejected": -225.10000610351562, + "loss": 0.591, + "rewards/accuracies": 0.7008241415023804, + "rewards/chosen": -0.34941405057907104, + "rewards/margins": 0.40869140625, + "rewards/rejected": -0.7582031488418579, + "step": 655 + }, + { + "epoch": 0.16992790937178168, + "grad_norm": 482.0, + "learning_rate": 4.1503604531410913e-07, + "logits/chosen": -2.65625, + "logits/rejected": -2.6312499046325684, + "logps/chosen": -361.20001220703125, + "logps/rejected": -381.6000061035156, + "loss": 0.4008, + "rewards/accuracies": 0.7942307591438293, + "rewards/chosen": -0.44111329317092896, + "rewards/margins": 1.216406226158142, + "rewards/rejected": -1.66015625, + "step": 660 + }, + { + "epoch": 0.17121524201853758, + "grad_norm": 812.0, + "learning_rate": 4.1439237899073123e-07, + "logits/chosen": -2.5250000953674316, + "logits/rejected": -2.4781250953674316, + "logps/chosen": -313.20001220703125, + "logps/rejected": -431.20001220703125, + "loss": 0.484, + "rewards/accuracies": 0.6697435975074768, + "rewards/chosen": -0.807812511920929, + "rewards/margins": 1.026953101158142, + "rewards/rejected": -1.8390624523162842, + "step": 665 + }, + { + "epoch": 0.1725025746652935, + "grad_norm": 422.0, + "learning_rate": 4.137487126673532e-07, + "logits/chosen": -2.659374952316284, + "logits/rejected": -2.382031202316284, + "logps/chosen": -253.8000030517578, + "logps/rejected": -277.0, + "loss": 0.5285, + "rewards/accuracies": 0.7028030157089233, + "rewards/chosen": -0.40742188692092896, + "rewards/margins": 0.7939453125, + "rewards/rejected": -1.200781226158142, + "step": 670 + }, + { + "epoch": 0.17378990731204944, + "grad_norm": 780.0, + "learning_rate": 4.1310504634397526e-07, + "logits/chosen": -2.621875047683716, + "logits/rejected": -2.643749952316284, + "logps/chosen": -374.0, + "logps/rejected": -327.6000061035156, + "loss": 0.557, + "rewards/accuracies": 0.7195237874984741, + "rewards/chosen": -0.44121092557907104, + "rewards/margins": 0.629687488079071, + "rewards/rejected": -1.071874976158142, + "step": 675 + }, + { + "epoch": 0.17507723995880536, + "grad_norm": 368.0, + "learning_rate": 4.124613800205973e-07, + "logits/chosen": -2.590625047683716, + "logits/rejected": -2.46875, + "logps/chosen": -352.20001220703125, + "logps/rejected": -456.3999938964844, + "loss": 0.4619, + "rewards/accuracies": 0.7571428418159485, + "rewards/chosen": -0.5796142816543579, + "rewards/margins": 1.122460961341858, + "rewards/rejected": -1.701562523841858, + "step": 680 + }, + { + "epoch": 0.17636457260556127, + "grad_norm": 432.0, + "learning_rate": 4.1181771369721935e-07, + "logits/chosen": -2.71875, + "logits/rejected": -2.768749952316284, + "logps/chosen": -345.6000061035156, + "logps/rejected": -322.3999938964844, + "loss": 0.6242, + "rewards/accuracies": 0.6142857670783997, + "rewards/chosen": -0.39667969942092896, + "rewards/margins": 0.5367187261581421, + "rewards/rejected": -0.9320312738418579, + "step": 685 + }, + { + "epoch": 0.1776519052523172, + "grad_norm": 348.0, + "learning_rate": 4.111740473738414e-07, + "logits/chosen": -2.721874952316284, + "logits/rejected": -2.637500047683716, + "logps/chosen": -348.79998779296875, + "logps/rejected": -424.0, + "loss": 0.6387, + "rewards/accuracies": 0.7215384244918823, + "rewards/chosen": -0.85546875, + "rewards/margins": 0.6181640625, + "rewards/rejected": -1.4734375476837158, + "step": 690 + }, + { + "epoch": 0.17893923789907312, + "grad_norm": 668.0, + "learning_rate": 4.1053038105046343e-07, + "logits/chosen": -2.7125000953674316, + "logits/rejected": -2.7281250953674316, + "logps/chosen": -378.79998779296875, + "logps/rejected": -396.79998779296875, + "loss": 0.5891, + "rewards/accuracies": 0.6688186526298523, + "rewards/chosen": -0.73828125, + "rewards/margins": 0.80859375, + "rewards/rejected": -1.546875, + "step": 695 + }, + { + "epoch": 0.18022657054582905, + "grad_norm": 748.0, + "learning_rate": 4.098867147270855e-07, + "logits/chosen": -2.590625047683716, + "logits/rejected": -2.731250047683716, + "logps/chosen": -313.79998779296875, + "logps/rejected": -345.3999938964844, + "loss": 0.5008, + "rewards/accuracies": 0.6567949056625366, + "rewards/chosen": -0.5291992425918579, + "rewards/margins": 0.8912109136581421, + "rewards/rejected": -1.4197266101837158, + "step": 700 + }, + { + "epoch": 0.18151390319258495, + "grad_norm": 696.0, + "learning_rate": 4.0924304840370747e-07, + "logits/chosen": -2.5374999046325684, + "logits/rejected": -2.6343750953674316, + "logps/chosen": -348.0, + "logps/rejected": -398.79998779296875, + "loss": 0.5875, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.842578113079071, + "rewards/margins": 0.7828124761581421, + "rewards/rejected": -1.6281249523162842, + "step": 705 + }, + { + "epoch": 0.18280123583934088, + "grad_norm": 584.0, + "learning_rate": 4.085993820803295e-07, + "logits/chosen": -2.768749952316284, + "logits/rejected": -2.753124952316284, + "logps/chosen": -353.20001220703125, + "logps/rejected": -373.6000061035156, + "loss": 0.6023, + "rewards/accuracies": 0.6217948794364929, + "rewards/chosen": -0.7132812738418579, + "rewards/margins": 0.408203125, + "rewards/rejected": -1.1203124523162842, + "step": 710 + }, + { + "epoch": 0.1840885684860968, + "grad_norm": 498.0, + "learning_rate": 4.079557157569516e-07, + "logits/chosen": -2.4140625, + "logits/rejected": -2.746875047683716, + "logps/chosen": -244.6999969482422, + "logps/rejected": -266.1000061035156, + "loss": 0.5406, + "rewards/accuracies": 0.6533333659172058, + "rewards/chosen": -0.093505859375, + "rewards/margins": 0.650585949420929, + "rewards/rejected": -0.744921863079071, + "step": 715 + }, + { + "epoch": 0.18537590113285274, + "grad_norm": 442.0, + "learning_rate": 4.073120494335736e-07, + "logits/chosen": -2.6937499046325684, + "logits/rejected": -2.653125047683716, + "logps/chosen": -324.3999938964844, + "logps/rejected": -358.79998779296875, + "loss": 0.4437, + "rewards/accuracies": 0.7633333802223206, + "rewards/chosen": -0.576953113079071, + "rewards/margins": 1.1906249523162842, + "rewards/rejected": -1.7703125476837158, + "step": 720 + }, + { + "epoch": 0.18666323377960864, + "grad_norm": 584.0, + "learning_rate": 4.0666838311019564e-07, + "logits/chosen": -2.4781250953674316, + "logits/rejected": -2.403125047683716, + "logps/chosen": -312.0, + "logps/rejected": -316.79998779296875, + "loss": 0.5188, + "rewards/accuracies": 0.734013557434082, + "rewards/chosen": -0.48579102754592896, + "rewards/margins": 0.805859386920929, + "rewards/rejected": -1.2921874523162842, + "step": 725 + }, + { + "epoch": 0.18795056642636457, + "grad_norm": 568.0, + "learning_rate": 4.0602471678681773e-07, + "logits/chosen": -2.518749952316284, + "logits/rejected": -2.4625000953674316, + "logps/chosen": -282.3999938964844, + "logps/rejected": -336.3999938964844, + "loss": 0.5508, + "rewards/accuracies": 0.6768590211868286, + "rewards/chosen": -0.35200196504592896, + "rewards/margins": 0.696093738079071, + "rewards/rejected": -1.0480468273162842, + "step": 730 + }, + { + "epoch": 0.1892378990731205, + "grad_norm": 540.0, + "learning_rate": 4.053810504634397e-07, + "logits/chosen": -2.565624952316284, + "logits/rejected": -2.5687499046325684, + "logps/chosen": -344.3999938964844, + "logps/rejected": -404.20001220703125, + "loss": 0.5355, + "rewards/accuracies": 0.6370862722396851, + "rewards/chosen": -0.31586915254592896, + "rewards/margins": 0.65057373046875, + "rewards/rejected": -0.9673827886581421, + "step": 735 + }, + { + "epoch": 0.19052523171987643, + "grad_norm": 540.0, + "learning_rate": 4.0473738414006176e-07, + "logits/chosen": -2.5609374046325684, + "logits/rejected": -2.5687499046325684, + "logps/chosen": -272.0, + "logps/rejected": -302.3999938964844, + "loss": 0.5609, + "rewards/accuracies": 0.6716667413711548, + "rewards/chosen": -0.5232177972793579, + "rewards/margins": 0.5865234136581421, + "rewards/rejected": -1.108984351158142, + "step": 740 + }, + { + "epoch": 0.19181256436663233, + "grad_norm": 308.0, + "learning_rate": 4.040937178166838e-07, + "logits/chosen": -2.606250047683716, + "logits/rejected": -2.7437500953674316, + "logps/chosen": -352.3999938964844, + "logps/rejected": -344.3999938964844, + "loss": 0.4395, + "rewards/accuracies": 0.7946212291717529, + "rewards/chosen": -0.522656261920929, + "rewards/margins": 1.072656273841858, + "rewards/rejected": -1.595312476158142, + "step": 745 + }, + { + "epoch": 0.19309989701338826, + "grad_norm": 506.0, + "learning_rate": 4.0345005149330585e-07, + "logits/chosen": -2.4453125, + "logits/rejected": -2.6031250953674316, + "logps/chosen": -312.79998779296875, + "logps/rejected": -315.3999938964844, + "loss": 0.5805, + "rewards/accuracies": 0.6192857027053833, + "rewards/chosen": -0.3548828065395355, + "rewards/margins": 0.48945313692092896, + "rewards/rejected": -0.844531238079071, + "step": 750 + }, + { + "epoch": 0.19438722966014418, + "grad_norm": 476.0, + "learning_rate": 4.028063851699279e-07, + "logits/chosen": -2.6156249046325684, + "logits/rejected": -2.765625, + "logps/chosen": -347.6000061035156, + "logps/rejected": -350.3999938964844, + "loss": 0.5609, + "rewards/accuracies": 0.6860606074333191, + "rewards/chosen": -0.490234375, + "rewards/margins": 0.4979492127895355, + "rewards/rejected": -0.990234375, + "step": 755 + }, + { + "epoch": 0.1956745623069001, + "grad_norm": 520.0, + "learning_rate": 4.0216271884654994e-07, + "logits/chosen": -2.471874952316284, + "logits/rejected": -2.5999999046325684, + "logps/chosen": -310.79998779296875, + "logps/rejected": -376.20001220703125, + "loss": 0.5156, + "rewards/accuracies": 0.7801190614700317, + "rewards/chosen": -0.7777343988418579, + "rewards/margins": 0.771484375, + "rewards/rejected": -1.5476562976837158, + "step": 760 + }, + { + "epoch": 0.196961894953656, + "grad_norm": 620.0, + "learning_rate": 4.01519052523172e-07, + "logits/chosen": -2.6968750953674316, + "logits/rejected": -2.6624999046325684, + "logps/chosen": -279.0, + "logps/rejected": -259.20001220703125, + "loss": 0.6156, + "rewards/accuracies": 0.5996825695037842, + "rewards/chosen": -0.3802734315395355, + "rewards/margins": 0.5007568597793579, + "rewards/rejected": -0.8807617425918579, + "step": 765 + }, + { + "epoch": 0.19824922760041194, + "grad_norm": 632.0, + "learning_rate": 4.00875386199794e-07, + "logits/chosen": -2.640625, + "logits/rejected": -2.7437500953674316, + "logps/chosen": -298.79998779296875, + "logps/rejected": -330.3999938964844, + "loss": 0.4701, + "rewards/accuracies": 0.7332723140716553, + "rewards/chosen": -0.2713867127895355, + "rewards/margins": 0.9898437261581421, + "rewards/rejected": -1.261328101158142, + "step": 770 + }, + { + "epoch": 0.19953656024716787, + "grad_norm": 592.0, + "learning_rate": 4.0023171987641606e-07, + "logits/chosen": -2.6156249046325684, + "logits/rejected": -2.71875, + "logps/chosen": -310.20001220703125, + "logps/rejected": -346.6000061035156, + "loss": 0.4914, + "rewards/accuracies": 0.7844444513320923, + "rewards/chosen": -0.45830076932907104, + "rewards/margins": 0.9039062261581421, + "rewards/rejected": -1.3624999523162842, + "step": 775 + }, + { + "epoch": 0.2008238928939238, + "grad_norm": 536.0, + "learning_rate": 3.995880535530381e-07, + "logits/chosen": -2.4765625, + "logits/rejected": -2.8812499046325684, + "logps/chosen": -291.3999938964844, + "logps/rejected": -322.79998779296875, + "loss": 0.5609, + "rewards/accuracies": 0.7338095903396606, + "rewards/chosen": -0.558886706829071, + "rewards/margins": 0.6539062261581421, + "rewards/rejected": -1.2117187976837158, + "step": 780 + }, + { + "epoch": 0.2021112255406797, + "grad_norm": 572.0, + "learning_rate": 3.989443872296601e-07, + "logits/chosen": -2.543750047683716, + "logits/rejected": -2.640625, + "logps/chosen": -284.6000061035156, + "logps/rejected": -323.0, + "loss": 0.5277, + "rewards/accuracies": 0.7286325097084045, + "rewards/chosen": -0.617480456829071, + "rewards/margins": 1.058203101158142, + "rewards/rejected": -1.673437476158142, + "step": 785 + }, + { + "epoch": 0.20339855818743563, + "grad_norm": 294.0, + "learning_rate": 3.9830072090628214e-07, + "logits/chosen": -2.2328124046325684, + "logits/rejected": -2.4281249046325684, + "logps/chosen": -315.20001220703125, + "logps/rejected": -334.6000061035156, + "loss": 0.4293, + "rewards/accuracies": 0.8843182325363159, + "rewards/chosen": -0.755078136920929, + "rewards/margins": 1.1140625476837158, + "rewards/rejected": -1.8703124523162842, + "step": 790 + }, + { + "epoch": 0.20468589083419156, + "grad_norm": 478.0, + "learning_rate": 3.9765705458290423e-07, + "logits/chosen": -2.765625, + "logits/rejected": -2.8125, + "logps/chosen": -319.3999938964844, + "logps/rejected": -313.6000061035156, + "loss": 0.5664, + "rewards/accuracies": 0.6896212697029114, + "rewards/chosen": -0.16733399033546448, + "rewards/margins": 0.5345703363418579, + "rewards/rejected": -0.7015625238418579, + "step": 795 + }, + { + "epoch": 0.2059732234809475, + "grad_norm": 438.0, + "learning_rate": 3.970133882595262e-07, + "logits/chosen": -2.5374999046325684, + "logits/rejected": -2.528125047683716, + "logps/chosen": -339.20001220703125, + "logps/rejected": -399.20001220703125, + "loss": 0.4535, + "rewards/accuracies": 0.7566666603088379, + "rewards/chosen": -0.792187511920929, + "rewards/margins": 1.0681641101837158, + "rewards/rejected": -1.8640625476837158, + "step": 800 + }, + { + "epoch": 0.2072605561277034, + "grad_norm": 478.0, + "learning_rate": 3.9636972193614827e-07, + "logits/chosen": -2.768749952316284, + "logits/rejected": -2.71875, + "logps/chosen": -326.6000061035156, + "logps/rejected": -358.3999938964844, + "loss": 0.4535, + "rewards/accuracies": 0.8113462328910828, + "rewards/chosen": -0.3396850526332855, + "rewards/margins": 1.009374976158142, + "rewards/rejected": -1.349218726158142, + "step": 805 + }, + { + "epoch": 0.20854788877445932, + "grad_norm": 396.0, + "learning_rate": 3.9572605561277036e-07, + "logits/chosen": -2.456249952316284, + "logits/rejected": -2.2874999046325684, + "logps/chosen": -314.0, + "logps/rejected": -382.79998779296875, + "loss": 0.452, + "rewards/accuracies": 0.7600001096725464, + "rewards/chosen": -0.664355456829071, + "rewards/margins": 0.964062511920929, + "rewards/rejected": -1.6328125, + "step": 810 + }, + { + "epoch": 0.20983522142121525, + "grad_norm": 376.0, + "learning_rate": 3.9508238928939235e-07, + "logits/chosen": -2.6968750953674316, + "logits/rejected": -2.395312547683716, + "logps/chosen": -280.20001220703125, + "logps/rejected": -321.6000061035156, + "loss": 0.4938, + "rewards/accuracies": 0.7758333683013916, + "rewards/chosen": -0.2733398377895355, + "rewards/margins": 0.8531249761581421, + "rewards/rejected": -1.1257812976837158, + "step": 815 + }, + { + "epoch": 0.21112255406797117, + "grad_norm": 338.0, + "learning_rate": 3.944387229660144e-07, + "logits/chosen": -2.6624999046325684, + "logits/rejected": -2.640625, + "logps/chosen": -264.3999938964844, + "logps/rejected": -267.6000061035156, + "loss": 0.523, + "rewards/accuracies": 0.730555534362793, + "rewards/chosen": -0.2965331971645355, + "rewards/margins": 0.895703136920929, + "rewards/rejected": -1.191796898841858, + "step": 820 + }, + { + "epoch": 0.21240988671472708, + "grad_norm": 458.0, + "learning_rate": 3.9379505664263644e-07, + "logits/chosen": -2.621875047683716, + "logits/rejected": -2.737499952316284, + "logps/chosen": -367.20001220703125, + "logps/rejected": -407.6000061035156, + "loss": 0.4633, + "rewards/accuracies": 0.7291209101676941, + "rewards/chosen": -0.5643066167831421, + "rewards/margins": 1.040624976158142, + "rewards/rejected": -1.605078101158142, + "step": 825 + }, + { + "epoch": 0.213697219361483, + "grad_norm": 620.0, + "learning_rate": 3.931513903192585e-07, + "logits/chosen": -2.809375047683716, + "logits/rejected": -2.703125, + "logps/chosen": -324.3999938964844, + "logps/rejected": -317.20001220703125, + "loss": 0.5516, + "rewards/accuracies": 0.6546037793159485, + "rewards/chosen": -0.2265625, + "rewards/margins": 0.5445312261581421, + "rewards/rejected": -0.7718750238418579, + "step": 830 + }, + { + "epoch": 0.21498455200823893, + "grad_norm": 388.0, + "learning_rate": 3.925077239958805e-07, + "logits/chosen": -2.621875047683716, + "logits/rejected": -2.721874952316284, + "logps/chosen": -288.20001220703125, + "logps/rejected": -316.3999938964844, + "loss": 0.4441, + "rewards/accuracies": 0.7594083547592163, + "rewards/chosen": -0.3755859434604645, + "rewards/margins": 0.9820312261581421, + "rewards/rejected": -1.3572266101837158, + "step": 835 + }, + { + "epoch": 0.21627188465499486, + "grad_norm": 928.0, + "learning_rate": 3.9186405767250257e-07, + "logits/chosen": -2.450000047683716, + "logits/rejected": -2.3968749046325684, + "logps/chosen": -309.3999938964844, + "logps/rejected": -300.8999938964844, + "loss": 0.5289, + "rewards/accuracies": 0.7225000262260437, + "rewards/chosen": -0.26640623807907104, + "rewards/margins": 0.860156238079071, + "rewards/rejected": -1.1259765625, + "step": 840 + }, + { + "epoch": 0.21755921730175076, + "grad_norm": 450.0, + "learning_rate": 3.912203913491246e-07, + "logits/chosen": -2.4312500953674316, + "logits/rejected": -2.403125047683716, + "logps/chosen": -306.20001220703125, + "logps/rejected": -376.3999938964844, + "loss": 0.4219, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.0867187976837158, + "rewards/margins": 1.255468726158142, + "rewards/rejected": -2.340625047683716, + "step": 845 + }, + { + "epoch": 0.2188465499485067, + "grad_norm": 448.0, + "learning_rate": 3.905767250257466e-07, + "logits/chosen": -2.543750047683716, + "logits/rejected": -2.753124952316284, + "logps/chosen": -276.8999938964844, + "logps/rejected": -279.20001220703125, + "loss": 0.5453, + "rewards/accuracies": 0.6954761743545532, + "rewards/chosen": -0.31816405057907104, + "rewards/margins": 0.560351550579071, + "rewards/rejected": -0.878125011920929, + "step": 850 + }, + { + "epoch": 0.22013388259526262, + "grad_norm": 524.0, + "learning_rate": 3.899330587023687e-07, + "logits/chosen": -2.5875000953674316, + "logits/rejected": -2.621875047683716, + "logps/chosen": -319.6000061035156, + "logps/rejected": -323.6000061035156, + "loss": 0.5316, + "rewards/accuracies": 0.6539285778999329, + "rewards/chosen": -0.25898438692092896, + "rewards/margins": 0.724609375, + "rewards/rejected": -0.984375, + "step": 855 + }, + { + "epoch": 0.22142121524201855, + "grad_norm": 616.0, + "learning_rate": 3.8928939237899074e-07, + "logits/chosen": -2.5687499046325684, + "logits/rejected": -2.628124952316284, + "logps/chosen": -341.20001220703125, + "logps/rejected": -335.79998779296875, + "loss": 0.4887, + "rewards/accuracies": 0.7632235288619995, + "rewards/chosen": -0.375, + "rewards/margins": 0.828125, + "rewards/rejected": -1.202539086341858, + "step": 860 + }, + { + "epoch": 0.22270854788877445, + "grad_norm": 486.0, + "learning_rate": 3.886457260556127e-07, + "logits/chosen": -2.6156249046325684, + "logits/rejected": -2.715625047683716, + "logps/chosen": -260.6000061035156, + "logps/rejected": -253.60000610351562, + "loss": 0.5559, + "rewards/accuracies": 0.7384341359138489, + "rewards/chosen": -0.23330077528953552, + "rewards/margins": 0.55078125, + "rewards/rejected": -0.785449206829071, + "step": 865 + }, + { + "epoch": 0.22399588053553038, + "grad_norm": 716.0, + "learning_rate": 3.8800205973223477e-07, + "logits/chosen": -2.6468749046325684, + "logits/rejected": -2.518749952316284, + "logps/chosen": -277.79998779296875, + "logps/rejected": -347.20001220703125, + "loss": 0.5953, + "rewards/accuracies": 0.6441666483879089, + "rewards/chosen": -0.5960937738418579, + "rewards/margins": 0.5267578363418579, + "rewards/rejected": -1.12109375, + "step": 870 + }, + { + "epoch": 0.2252832131822863, + "grad_norm": 556.0, + "learning_rate": 3.8735839340885686e-07, + "logits/chosen": -2.528125047683716, + "logits/rejected": -2.5, + "logps/chosen": -302.3999938964844, + "logps/rejected": -432.0, + "loss": 0.3428, + "rewards/accuracies": 0.8178571462631226, + "rewards/chosen": -0.7222656011581421, + "rewards/margins": 1.732812523841858, + "rewards/rejected": -2.453125, + "step": 875 + }, + { + "epoch": 0.22657054582904224, + "grad_norm": 808.0, + "learning_rate": 3.8671472708547885e-07, + "logits/chosen": -2.706249952316284, + "logits/rejected": -2.690624952316284, + "logps/chosen": -300.29998779296875, + "logps/rejected": -271.6000061035156, + "loss": 0.6844, + "rewards/accuracies": 0.6163309216499329, + "rewards/chosen": -0.6363769769668579, + "rewards/margins": 0.41777342557907104, + "rewards/rejected": -1.0519530773162842, + "step": 880 + }, + { + "epoch": 0.22785787847579814, + "grad_norm": 700.0, + "learning_rate": 3.860710607621009e-07, + "logits/chosen": -2.5062499046325684, + "logits/rejected": -2.5078125, + "logps/chosen": -341.3999938964844, + "logps/rejected": -378.79998779296875, + "loss": 0.4844, + "rewards/accuracies": 0.7562013864517212, + "rewards/chosen": -0.6080077886581421, + "rewards/margins": 1.0304687023162842, + "rewards/rejected": -1.6398437023162842, + "step": 885 + }, + { + "epoch": 0.22914521112255407, + "grad_norm": 2400.0, + "learning_rate": 3.8542739443872294e-07, + "logits/chosen": -2.421875, + "logits/rejected": -2.5093750953674316, + "logps/chosen": -328.79998779296875, + "logps/rejected": -372.79998779296875, + "loss": 0.5613, + "rewards/accuracies": 0.665357232093811, + "rewards/chosen": -1.1453125476837158, + "rewards/margins": 1.05859375, + "rewards/rejected": -2.203906297683716, + "step": 890 + }, + { + "epoch": 0.23043254376931, + "grad_norm": 360.0, + "learning_rate": 3.84783728115345e-07, + "logits/chosen": -2.4468750953674316, + "logits/rejected": -2.46875, + "logps/chosen": -264.0, + "logps/rejected": -323.6000061035156, + "loss": 0.502, + "rewards/accuracies": 0.6850524544715881, + "rewards/chosen": -0.668017566204071, + "rewards/margins": 0.983203113079071, + "rewards/rejected": -1.6515624523162842, + "step": 895 + }, + { + "epoch": 0.23171987641606592, + "grad_norm": 536.0, + "learning_rate": 3.84140061791967e-07, + "logits/chosen": -2.621875047683716, + "logits/rejected": -2.4906249046325684, + "logps/chosen": -337.79998779296875, + "logps/rejected": -362.3999938964844, + "loss": 0.3766, + "rewards/accuracies": 0.8441667556762695, + "rewards/chosen": -0.7457031011581421, + "rewards/margins": 1.381250023841858, + "rewards/rejected": -2.1265625953674316, + "step": 900 + }, + { + "epoch": 0.23300720906282182, + "grad_norm": 768.0, + "learning_rate": 3.8349639546858907e-07, + "logits/chosen": -2.53125, + "logits/rejected": -2.703125, + "logps/chosen": -338.3999938964844, + "logps/rejected": -332.6499938964844, + "loss": 0.602, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.0268065929412842, + "rewards/margins": 0.6792968511581421, + "rewards/rejected": -1.70654296875, + "step": 905 + }, + { + "epoch": 0.23429454170957775, + "grad_norm": 652.0, + "learning_rate": 3.828527291452111e-07, + "logits/chosen": -2.643749952316284, + "logits/rejected": -2.721874952316284, + "logps/chosen": -335.20001220703125, + "logps/rejected": -372.0, + "loss": 0.5449, + "rewards/accuracies": 0.6625458002090454, + "rewards/chosen": -0.15175780653953552, + "rewards/margins": 0.698437511920929, + "rewards/rejected": -0.850390613079071, + "step": 910 + }, + { + "epoch": 0.23558187435633368, + "grad_norm": 410.0, + "learning_rate": 3.8220906282183315e-07, + "logits/chosen": -2.418750047683716, + "logits/rejected": -2.3578124046325684, + "logps/chosen": -228.3000030517578, + "logps/rejected": -329.0, + "loss": 0.5254, + "rewards/accuracies": 0.6444138884544373, + "rewards/chosen": -0.29680174589157104, + "rewards/margins": 0.859375, + "rewards/rejected": -1.15625, + "step": 915 + }, + { + "epoch": 0.2368692070030896, + "grad_norm": 564.0, + "learning_rate": 3.815653964984552e-07, + "logits/chosen": -2.721874952316284, + "logits/rejected": -2.784374952316284, + "logps/chosen": -232.60000610351562, + "logps/rejected": -217.10000610351562, + "loss": 0.6203, + "rewards/accuracies": 0.5828355550765991, + "rewards/chosen": -0.4609619081020355, + "rewards/margins": 0.4124511778354645, + "rewards/rejected": -0.8729248046875, + "step": 920 + }, + { + "epoch": 0.2381565396498455, + "grad_norm": 660.0, + "learning_rate": 3.8092173017507724e-07, + "logits/chosen": -2.4937500953674316, + "logits/rejected": -2.559375047683716, + "logps/chosen": -260.0, + "logps/rejected": -340.0, + "loss": 0.5184, + "rewards/accuracies": 0.7143589854240417, + "rewards/chosen": -0.43195801973342896, + "rewards/margins": 0.904296875, + "rewards/rejected": -1.3328125476837158, + "step": 925 + }, + { + "epoch": 0.23944387229660144, + "grad_norm": 620.0, + "learning_rate": 3.8027806385169923e-07, + "logits/chosen": -2.534374952316284, + "logits/rejected": -2.643749952316284, + "logps/chosen": -306.20001220703125, + "logps/rejected": -346.25, + "loss": 0.5262, + "rewards/accuracies": 0.7546428442001343, + "rewards/chosen": -0.9847167730331421, + "rewards/margins": 1.0115478038787842, + "rewards/rejected": -1.997802734375, + "step": 930 + }, + { + "epoch": 0.24073120494335737, + "grad_norm": 608.0, + "learning_rate": 3.796343975283213e-07, + "logits/chosen": -2.606250047683716, + "logits/rejected": -2.6187500953674316, + "logps/chosen": -297.79998779296875, + "logps/rejected": -330.20001220703125, + "loss": 0.7281, + "rewards/accuracies": 0.5977020859718323, + "rewards/chosen": -0.8570312261581421, + "rewards/margins": 0.4725585877895355, + "rewards/rejected": -1.3273437023162842, + "step": 935 + }, + { + "epoch": 0.2420185375901133, + "grad_norm": 824.0, + "learning_rate": 3.7899073120494337e-07, + "logits/chosen": -2.4593749046325684, + "logits/rejected": -2.4703125953674316, + "logps/chosen": -269.3999938964844, + "logps/rejected": -304.3999938964844, + "loss": 0.5426, + "rewards/accuracies": 0.7101190686225891, + "rewards/chosen": -0.36250001192092896, + "rewards/margins": 0.7572265863418579, + "rewards/rejected": -1.1183593273162842, + "step": 940 + }, + { + "epoch": 0.2433058702368692, + "grad_norm": 308.0, + "learning_rate": 3.7834706488156536e-07, + "logits/chosen": -2.690624952316284, + "logits/rejected": -2.731250047683716, + "logps/chosen": -295.0, + "logps/rejected": -271.6000061035156, + "loss": 0.6297, + "rewards/accuracies": 0.6891775131225586, + "rewards/chosen": -0.2513671815395355, + "rewards/margins": 0.383544921875, + "rewards/rejected": -0.635546863079071, + "step": 945 + }, + { + "epoch": 0.24459320288362513, + "grad_norm": 556.0, + "learning_rate": 3.777033985581874e-07, + "logits/chosen": -2.5374999046325684, + "logits/rejected": -2.440624952316284, + "logps/chosen": -262.0, + "logps/rejected": -327.3999938964844, + "loss": 0.4922, + "rewards/accuracies": 0.7474650144577026, + "rewards/chosen": -0.48652344942092896, + "rewards/margins": 0.8828125, + "rewards/rejected": -1.368749976158142, + "step": 950 + }, + { + "epoch": 0.24588053553038106, + "grad_norm": 370.0, + "learning_rate": 3.770597322348095e-07, + "logits/chosen": -2.471874952316284, + "logits/rejected": -2.390625, + "logps/chosen": -297.20001220703125, + "logps/rejected": -350.79998779296875, + "loss": 0.5125, + "rewards/accuracies": 0.7304762601852417, + "rewards/chosen": -0.760937511920929, + "rewards/margins": 1.004296898841858, + "rewards/rejected": -1.763281226158142, + "step": 955 + }, + { + "epoch": 0.24716786817713698, + "grad_norm": 376.0, + "learning_rate": 3.764160659114315e-07, + "logits/chosen": -2.5875000953674316, + "logits/rejected": -2.5093750953674316, + "logps/chosen": -384.79998779296875, + "logps/rejected": -418.0, + "loss": 0.4014, + "rewards/accuracies": 0.8091667294502258, + "rewards/chosen": -0.545703113079071, + "rewards/margins": 1.2062499523162842, + "rewards/rejected": -1.75390625, + "step": 960 + }, + { + "epoch": 0.24845520082389289, + "grad_norm": 596.0, + "learning_rate": 3.7577239958805353e-07, + "logits/chosen": -2.465625047683716, + "logits/rejected": -2.432812452316284, + "logps/chosen": -266.79998779296875, + "logps/rejected": -294.3999938964844, + "loss": 0.466, + "rewards/accuracies": 0.7658333778381348, + "rewards/chosen": -0.26995849609375, + "rewards/margins": 0.9984375238418579, + "rewards/rejected": -1.266210913658142, + "step": 965 + }, + { + "epoch": 0.24974253347064881, + "grad_norm": 382.0, + "learning_rate": 3.7512873326467557e-07, + "logits/chosen": -2.590625047683716, + "logits/rejected": -2.7281250953674316, + "logps/chosen": -369.20001220703125, + "logps/rejected": -412.0, + "loss": 0.4752, + "rewards/accuracies": 0.7209615111351013, + "rewards/chosen": -0.7300781011581421, + "rewards/margins": 1.1121094226837158, + "rewards/rejected": -1.844140648841858, + "step": 970 + }, + { + "epoch": 0.25102986611740474, + "grad_norm": 294.0, + "learning_rate": 3.744850669412976e-07, + "logits/chosen": -2.6875, + "logits/rejected": -2.6937499046325684, + "logps/chosen": -341.20001220703125, + "logps/rejected": -342.0, + "loss": 0.4449, + "rewards/accuracies": 0.7717857956886292, + "rewards/chosen": -0.3406738340854645, + "rewards/margins": 1.075781226158142, + "rewards/rejected": -1.416406273841858, + "step": 975 + }, + { + "epoch": 0.25231719876416064, + "grad_norm": 458.0, + "learning_rate": 3.7384140061791965e-07, + "logits/chosen": -2.674999952316284, + "logits/rejected": -2.768749952316284, + "logps/chosen": -339.6000061035156, + "logps/rejected": -339.20001220703125, + "loss": 0.5305, + "rewards/accuracies": 0.727857232093811, + "rewards/chosen": -0.5103515386581421, + "rewards/margins": 0.6634765863418579, + "rewards/rejected": -1.171875, + "step": 980 + }, + { + "epoch": 0.2536045314109166, + "grad_norm": 596.0, + "learning_rate": 3.731977342945417e-07, + "logits/chosen": -2.5374999046325684, + "logits/rejected": -2.5718750953674316, + "logps/chosen": -362.3999938964844, + "logps/rejected": -413.0, + "loss": 0.5141, + "rewards/accuracies": 0.6727563738822937, + "rewards/chosen": -1.088281273841858, + "rewards/margins": 0.917919933795929, + "rewards/rejected": -2.0101561546325684, + "step": 985 + }, + { + "epoch": 0.2548918640576725, + "grad_norm": 324.0, + "learning_rate": 3.7255406797116374e-07, + "logits/chosen": -2.4000000953674316, + "logits/rejected": -2.2203125953674316, + "logps/chosen": -332.3999938964844, + "logps/rejected": -443.6000061035156, + "loss": 0.3072, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -0.721875011920929, + "rewards/margins": 1.8125, + "rewards/rejected": -2.53125, + "step": 990 + }, + { + "epoch": 0.2561791967044284, + "grad_norm": 388.0, + "learning_rate": 3.7191040164778573e-07, + "logits/chosen": -2.424999952316284, + "logits/rejected": -2.5406250953674316, + "logps/chosen": -346.79998779296875, + "logps/rejected": -381.20001220703125, + "loss": 0.5512, + "rewards/accuracies": 0.7433333396911621, + "rewards/chosen": -0.932812511920929, + "rewards/margins": 0.842968761920929, + "rewards/rejected": -1.7765624523162842, + "step": 995 + }, + { + "epoch": 0.25746652935118436, + "grad_norm": 506.0, + "learning_rate": 3.712667353244078e-07, + "logits/chosen": -2.5875000953674316, + "logits/rejected": -2.6031250953674316, + "logps/chosen": -294.3999938964844, + "logps/rejected": -357.6000061035156, + "loss": 0.5066, + "rewards/accuracies": 0.7355769276618958, + "rewards/chosen": -0.6908203363418579, + "rewards/margins": 0.907031238079071, + "rewards/rejected": -1.5984375476837158, + "step": 1000 + }, + { + "epoch": 0.25875386199794026, + "grad_norm": 258.0, + "learning_rate": 3.7062306900102987e-07, + "logits/chosen": -2.6937499046325684, + "logits/rejected": -2.65625, + "logps/chosen": -315.0, + "logps/rejected": -325.3999938964844, + "loss": 0.4715, + "rewards/accuracies": 0.7408424615859985, + "rewards/chosen": -0.541210949420929, + "rewards/margins": 0.794921875, + "rewards/rejected": -1.337499976158142, + "step": 1005 + }, + { + "epoch": 0.2600411946446962, + "grad_norm": 392.0, + "learning_rate": 3.6997940267765186e-07, + "logits/chosen": -2.746875047683716, + "logits/rejected": -2.590625047683716, + "logps/chosen": -373.6000061035156, + "logps/rejected": -358.79998779296875, + "loss": 0.4383, + "rewards/accuracies": 0.799945056438446, + "rewards/chosen": -0.27763670682907104, + "rewards/margins": 1.05859375, + "rewards/rejected": -1.3351562023162842, + "step": 1010 + }, + { + "epoch": 0.2613285272914521, + "grad_norm": 364.0, + "learning_rate": 3.6933573635427395e-07, + "logits/chosen": -2.609375, + "logits/rejected": -2.668750047683716, + "logps/chosen": -349.20001220703125, + "logps/rejected": -326.20001220703125, + "loss": 0.5035, + "rewards/accuracies": 0.7708333730697632, + "rewards/chosen": -0.22158202528953552, + "rewards/margins": 0.6683593988418579, + "rewards/rejected": -0.8910156488418579, + "step": 1015 + }, + { + "epoch": 0.262615859938208, + "grad_norm": 392.0, + "learning_rate": 3.68692070030896e-07, + "logits/chosen": -2.542187452316284, + "logits/rejected": -2.4828124046325684, + "logps/chosen": -328.0, + "logps/rejected": -410.79998779296875, + "loss": 0.4078, + "rewards/accuracies": 0.8108333349227905, + "rewards/chosen": -0.6392577886581421, + "rewards/margins": 1.032812476158142, + "rewards/rejected": -1.6749999523162842, + "step": 1020 + }, + { + "epoch": 0.263903192584964, + "grad_norm": 1048.0, + "learning_rate": 3.68048403707518e-07, + "logits/chosen": -2.565624952316284, + "logits/rejected": -2.5718750953674316, + "logps/chosen": -275.0, + "logps/rejected": -270.29998779296875, + "loss": 0.4949, + "rewards/accuracies": 0.6901190876960754, + "rewards/chosen": -0.42011719942092896, + "rewards/margins": 0.793749988079071, + "rewards/rejected": -1.2126953601837158, + "step": 1025 + }, + { + "epoch": 0.2651905252317199, + "grad_norm": 396.0, + "learning_rate": 3.6740473738414003e-07, + "logits/chosen": -2.606250047683716, + "logits/rejected": -2.7249999046325684, + "logps/chosen": -302.79998779296875, + "logps/rejected": -346.6000061035156, + "loss": 0.4789, + "rewards/accuracies": 0.7222222089767456, + "rewards/chosen": -0.436767578125, + "rewards/margins": 0.9664062261581421, + "rewards/rejected": -1.40234375, + "step": 1030 + }, + { + "epoch": 0.2664778578784758, + "grad_norm": 544.0, + "learning_rate": 3.6676107106076207e-07, + "logits/chosen": -2.625, + "logits/rejected": -2.4984374046325684, + "logps/chosen": -241.0, + "logps/rejected": -261.6000061035156, + "loss": 0.5227, + "rewards/accuracies": 0.7159091234207153, + "rewards/chosen": -0.3323608338832855, + "rewards/margins": 0.635937511920929, + "rewards/rejected": -0.967968761920929, + "step": 1035 + }, + { + "epoch": 0.26776519052523173, + "grad_norm": 438.0, + "learning_rate": 3.661174047373841e-07, + "logits/chosen": -2.653125047683716, + "logits/rejected": -2.515625, + "logps/chosen": -322.79998779296875, + "logps/rejected": -386.79998779296875, + "loss": 0.4051, + "rewards/accuracies": 0.8045238256454468, + "rewards/chosen": -0.669384777545929, + "rewards/margins": 1.2625000476837158, + "rewards/rejected": -1.9304687976837158, + "step": 1040 + }, + { + "epoch": 0.26905252317198763, + "grad_norm": 438.0, + "learning_rate": 3.6547373841400616e-07, + "logits/chosen": -2.6312499046325684, + "logits/rejected": -2.481250047683716, + "logps/chosen": -373.20001220703125, + "logps/rejected": -369.0, + "loss": 0.4598, + "rewards/accuracies": 0.7609615921974182, + "rewards/chosen": -0.584912121295929, + "rewards/margins": 0.94921875, + "rewards/rejected": -1.5339844226837158, + "step": 1045 + }, + { + "epoch": 0.2703398558187436, + "grad_norm": 316.0, + "learning_rate": 3.648300720906282e-07, + "logits/chosen": -2.668750047683716, + "logits/rejected": -2.674999952316284, + "logps/chosen": -265.79998779296875, + "logps/rejected": -307.79998779296875, + "loss": 0.4527, + "rewards/accuracies": 0.7421795129776001, + "rewards/chosen": -0.39453125, + "rewards/margins": 1.0041015148162842, + "rewards/rejected": -1.3976562023162842, + "step": 1050 + }, + { + "epoch": 0.2716271884654995, + "grad_norm": 584.0, + "learning_rate": 3.6418640576725024e-07, + "logits/chosen": -2.643749952316284, + "logits/rejected": -2.6937499046325684, + "logps/chosen": -292.20001220703125, + "logps/rejected": -385.20001220703125, + "loss": 0.509, + "rewards/accuracies": 0.7285714745521545, + "rewards/chosen": -0.5882812738418579, + "rewards/margins": 0.8628906011581421, + "rewards/rejected": -1.453125, + "step": 1055 + }, + { + "epoch": 0.2729145211122554, + "grad_norm": 584.0, + "learning_rate": 3.635427394438723e-07, + "logits/chosen": -2.753124952316284, + "logits/rejected": -2.715625047683716, + "logps/chosen": -281.0, + "logps/rejected": -348.79998779296875, + "loss": 0.4887, + "rewards/accuracies": 0.729455292224884, + "rewards/chosen": -0.77978515625, + "rewards/margins": 0.9566406011581421, + "rewards/rejected": -1.735937476158142, + "step": 1060 + }, + { + "epoch": 0.27420185375901135, + "grad_norm": 452.0, + "learning_rate": 3.6289907312049433e-07, + "logits/chosen": -2.7125000953674316, + "logits/rejected": -2.628124952316284, + "logps/chosen": -324.3999938964844, + "logps/rejected": -378.79998779296875, + "loss": 0.5367, + "rewards/accuracies": 0.675595223903656, + "rewards/chosen": -0.6441406011581421, + "rewards/margins": 0.842968761920929, + "rewards/rejected": -1.48828125, + "step": 1065 + }, + { + "epoch": 0.27548918640576725, + "grad_norm": 588.0, + "learning_rate": 3.6225540679711637e-07, + "logits/chosen": -2.549999952316284, + "logits/rejected": -2.278125047683716, + "logps/chosen": -230.3000030517578, + "logps/rejected": -278.0, + "loss": 0.5535, + "rewards/accuracies": 0.6388888955116272, + "rewards/chosen": -0.658886730670929, + "rewards/margins": 0.9332031011581421, + "rewards/rejected": -1.591406226158142, + "step": 1070 + }, + { + "epoch": 0.27677651905252315, + "grad_norm": 390.0, + "learning_rate": 3.6161174047373836e-07, + "logits/chosen": -2.606250047683716, + "logits/rejected": -2.549999952316284, + "logps/chosen": -354.3999938964844, + "logps/rejected": -407.20001220703125, + "loss": 0.4848, + "rewards/accuracies": 0.7484615445137024, + "rewards/chosen": -0.9222656488418579, + "rewards/margins": 1.1101562976837158, + "rewards/rejected": -2.0328125953674316, + "step": 1075 + }, + { + "epoch": 0.2780638516992791, + "grad_norm": 398.0, + "learning_rate": 3.6096807415036046e-07, + "logits/chosen": -2.653125047683716, + "logits/rejected": -2.6875, + "logps/chosen": -271.6000061035156, + "logps/rejected": -331.0, + "loss": 0.3867, + "rewards/accuracies": 0.7645604610443115, + "rewards/chosen": -0.49492186307907104, + "rewards/margins": 1.3328125476837158, + "rewards/rejected": -1.828125, + "step": 1080 + }, + { + "epoch": 0.279351184346035, + "grad_norm": 324.0, + "learning_rate": 3.603244078269825e-07, + "logits/chosen": -2.684375047683716, + "logits/rejected": -2.65625, + "logps/chosen": -398.0, + "logps/rejected": -371.79998779296875, + "loss": 0.4914, + "rewards/accuracies": 0.7226282358169556, + "rewards/chosen": -0.6884765625, + "rewards/margins": 0.9820312261581421, + "rewards/rejected": -1.670507788658142, + "step": 1085 + }, + { + "epoch": 0.2806385169927909, + "grad_norm": 358.0, + "learning_rate": 3.596807415036045e-07, + "logits/chosen": -2.640625, + "logits/rejected": -2.674999952316284, + "logps/chosen": -257.29998779296875, + "logps/rejected": -262.0, + "loss": 0.5773, + "rewards/accuracies": 0.577445924282074, + "rewards/chosen": -0.054645538330078125, + "rewards/margins": 0.522167980670929, + "rewards/rejected": -0.57684326171875, + "step": 1090 + }, + { + "epoch": 0.28192584963954687, + "grad_norm": 414.0, + "learning_rate": 3.590370751802266e-07, + "logits/chosen": -2.546875, + "logits/rejected": -2.5546875, + "logps/chosen": -275.20001220703125, + "logps/rejected": -302.79998779296875, + "loss": 0.5402, + "rewards/accuracies": 0.5860714912414551, + "rewards/chosen": -0.537109375, + "rewards/margins": 0.756640613079071, + "rewards/rejected": -1.29296875, + "step": 1095 + }, + { + "epoch": 0.28321318228630277, + "grad_norm": 516.0, + "learning_rate": 3.583934088568486e-07, + "logits/chosen": -2.6624999046325684, + "logits/rejected": -2.6343750953674316, + "logps/chosen": -336.3999938964844, + "logps/rejected": -346.3999938964844, + "loss": 0.4574, + "rewards/accuracies": 0.7988095283508301, + "rewards/chosen": -0.504687488079071, + "rewards/margins": 0.9296875, + "rewards/rejected": -1.435937523841858, + "step": 1100 + }, + { + "epoch": 0.2845005149330587, + "grad_norm": 346.0, + "learning_rate": 3.577497425334706e-07, + "logits/chosen": -2.5250000953674316, + "logits/rejected": -2.465625047683716, + "logps/chosen": -270.6000061035156, + "logps/rejected": -335.20001220703125, + "loss": 0.4553, + "rewards/accuracies": 0.7442308664321899, + "rewards/chosen": -0.626953125, + "rewards/margins": 1.021093726158142, + "rewards/rejected": -1.6492187976837158, + "step": 1105 + }, + { + "epoch": 0.2857878475798146, + "grad_norm": 908.0, + "learning_rate": 3.571060762100927e-07, + "logits/chosen": -2.7281250953674316, + "logits/rejected": -2.7718749046325684, + "logps/chosen": -311.0, + "logps/rejected": -281.0, + "loss": 0.6199, + "rewards/accuracies": 0.621666669845581, + "rewards/chosen": -0.722851574420929, + "rewards/margins": 0.544921875, + "rewards/rejected": -1.26953125, + "step": 1110 + }, + { + "epoch": 0.2870751802265705, + "grad_norm": 652.0, + "learning_rate": 3.564624098867147e-07, + "logits/chosen": -2.3375000953674316, + "logits/rejected": -2.4046874046325684, + "logps/chosen": -273.20001220703125, + "logps/rejected": -303.20001220703125, + "loss": 0.5391, + "rewards/accuracies": 0.7273626327514648, + "rewards/chosen": -0.537109375, + "rewards/margins": 0.9214843511581421, + "rewards/rejected": -1.4578125476837158, + "step": 1115 + }, + { + "epoch": 0.2883625128733265, + "grad_norm": 486.0, + "learning_rate": 3.5581874356333674e-07, + "logits/chosen": -2.575000047683716, + "logits/rejected": -2.5843749046325684, + "logps/chosen": -280.6000061035156, + "logps/rejected": -348.0, + "loss": 0.4984, + "rewards/accuracies": 0.8272619247436523, + "rewards/chosen": -0.6622070074081421, + "rewards/margins": 1.033593773841858, + "rewards/rejected": -1.694921851158142, + "step": 1120 + }, + { + "epoch": 0.2896498455200824, + "grad_norm": 580.0, + "learning_rate": 3.551750772399588e-07, + "logits/chosen": -2.5687499046325684, + "logits/rejected": -2.653125047683716, + "logps/chosen": -304.20001220703125, + "logps/rejected": -304.0, + "loss": 0.5305, + "rewards/accuracies": 0.7392732501029968, + "rewards/chosen": -0.556640625, + "rewards/margins": 0.849609375, + "rewards/rejected": -1.40625, + "step": 1125 + }, + { + "epoch": 0.2909371781668383, + "grad_norm": 644.0, + "learning_rate": 3.5453141091658083e-07, + "logits/chosen": -2.510937452316284, + "logits/rejected": -2.668750047683716, + "logps/chosen": -293.0, + "logps/rejected": -310.1000061035156, + "loss": 0.5764, + "rewards/accuracies": 0.6741666197776794, + "rewards/chosen": -0.819140613079071, + "rewards/margins": 0.8248046636581421, + "rewards/rejected": -1.6435058116912842, + "step": 1130 + }, + { + "epoch": 0.29222451081359424, + "grad_norm": 470.0, + "learning_rate": 3.5388774459320287e-07, + "logits/chosen": -2.6312499046325684, + "logits/rejected": -2.528125047683716, + "logps/chosen": -276.3999938964844, + "logps/rejected": -365.20001220703125, + "loss": 0.4465, + "rewards/accuracies": 0.7272436022758484, + "rewards/chosen": -0.36796873807907104, + "rewards/margins": 1.1328125, + "rewards/rejected": -1.502343773841858, + "step": 1135 + }, + { + "epoch": 0.29351184346035014, + "grad_norm": 510.0, + "learning_rate": 3.5324407826982486e-07, + "logits/chosen": -2.5531249046325684, + "logits/rejected": -2.6187500953674316, + "logps/chosen": -296.6000061035156, + "logps/rejected": -297.3999938964844, + "loss": 0.5246, + "rewards/accuracies": 0.6811147928237915, + "rewards/chosen": -0.581250011920929, + "rewards/margins": 0.866406261920929, + "rewards/rejected": -1.4445312023162842, + "step": 1140 + }, + { + "epoch": 0.2947991761071061, + "grad_norm": 512.0, + "learning_rate": 3.5260041194644696e-07, + "logits/chosen": -2.778125047683716, + "logits/rejected": -2.6656250953674316, + "logps/chosen": -371.6000061035156, + "logps/rejected": -386.3999938964844, + "loss": 0.6312, + "rewards/accuracies": 0.6083333492279053, + "rewards/chosen": -0.772656261920929, + "rewards/margins": 0.5435546636581421, + "rewards/rejected": -1.31640625, + "step": 1145 + }, + { + "epoch": 0.296086508753862, + "grad_norm": 688.0, + "learning_rate": 3.51956745623069e-07, + "logits/chosen": -2.403125047683716, + "logits/rejected": -2.301562547683716, + "logps/chosen": -377.20001220703125, + "logps/rejected": -365.6000061035156, + "loss": 0.4875, + "rewards/accuracies": 0.7785714864730835, + "rewards/chosen": -0.807421863079071, + "rewards/margins": 0.875, + "rewards/rejected": -1.685937523841858, + "step": 1150 + }, + { + "epoch": 0.2973738414006179, + "grad_norm": 382.0, + "learning_rate": 3.51313079299691e-07, + "logits/chosen": -2.5999999046325684, + "logits/rejected": -2.4593749046325684, + "logps/chosen": -324.20001220703125, + "logps/rejected": -374.79998779296875, + "loss": 0.4535, + "rewards/accuracies": 0.8120238184928894, + "rewards/chosen": -0.852734386920929, + "rewards/margins": 1.110937476158142, + "rewards/rejected": -1.962499976158142, + "step": 1155 + }, + { + "epoch": 0.29866117404737386, + "grad_norm": 348.0, + "learning_rate": 3.506694129763131e-07, + "logits/chosen": -2.4281249046325684, + "logits/rejected": -2.356250047683716, + "logps/chosen": -328.0, + "logps/rejected": -376.5, + "loss": 0.4484, + "rewards/accuracies": 0.7140909433364868, + "rewards/chosen": -1.1007812023162842, + "rewards/margins": 1.4464843273162842, + "rewards/rejected": -2.5492186546325684, + "step": 1160 + }, + { + "epoch": 0.29994850669412976, + "grad_norm": 676.0, + "learning_rate": 3.5002574665293513e-07, + "logits/chosen": -2.5062499046325684, + "logits/rejected": -2.6468749046325684, + "logps/chosen": -370.79998779296875, + "logps/rejected": -340.0, + "loss": 0.6035, + "rewards/accuracies": 0.6631169319152832, + "rewards/chosen": -0.813671886920929, + "rewards/margins": 0.6452392339706421, + "rewards/rejected": -1.458593726158142, + "step": 1165 + }, + { + "epoch": 0.30123583934088566, + "grad_norm": 470.0, + "learning_rate": 3.493820803295571e-07, + "logits/chosen": -2.668750047683716, + "logits/rejected": -2.706249952316284, + "logps/chosen": -362.3999938964844, + "logps/rejected": -426.79998779296875, + "loss": 0.5387, + "rewards/accuracies": 0.705833375453949, + "rewards/chosen": -0.4727218747138977, + "rewards/margins": 0.8597656488418579, + "rewards/rejected": -1.33203125, + "step": 1170 + }, + { + "epoch": 0.3025231719876416, + "grad_norm": 404.0, + "learning_rate": 3.487384140061792e-07, + "logits/chosen": -2.6468749046325684, + "logits/rejected": -2.6812500953674316, + "logps/chosen": -323.79998779296875, + "logps/rejected": -352.6000061035156, + "loss": 0.5383, + "rewards/accuracies": 0.6920453906059265, + "rewards/chosen": -0.542187511920929, + "rewards/margins": 0.7816406488418579, + "rewards/rejected": -1.322265625, + "step": 1175 + }, + { + "epoch": 0.3038105046343975, + "grad_norm": 1008.0, + "learning_rate": 3.480947476828012e-07, + "logits/chosen": -2.53125, + "logits/rejected": -2.643749952316284, + "logps/chosen": -358.0, + "logps/rejected": -343.0, + "loss": 0.5473, + "rewards/accuracies": 0.7148352265357971, + "rewards/chosen": -0.9234374761581421, + "rewards/margins": 0.979687511920929, + "rewards/rejected": -1.9000122547149658, + "step": 1180 + }, + { + "epoch": 0.30509783728115347, + "grad_norm": 512.0, + "learning_rate": 3.4745108135942325e-07, + "logits/chosen": -2.4906249046325684, + "logits/rejected": -2.5374999046325684, + "logps/chosen": -371.20001220703125, + "logps/rejected": -364.20001220703125, + "loss": 0.4797, + "rewards/accuracies": 0.7716484069824219, + "rewards/chosen": -0.614086925983429, + "rewards/margins": 1.067895531654358, + "rewards/rejected": -1.681265115737915, + "step": 1185 + }, + { + "epoch": 0.3063851699279094, + "grad_norm": 410.0, + "learning_rate": 3.4680741503604534e-07, + "logits/chosen": -2.559375047683716, + "logits/rejected": -2.59375, + "logps/chosen": -363.6000061035156, + "logps/rejected": -414.20001220703125, + "loss": 0.5047, + "rewards/accuracies": 0.7587546110153198, + "rewards/chosen": -0.812548816204071, + "rewards/margins": 0.8949218988418579, + "rewards/rejected": -1.708593726158142, + "step": 1190 + }, + { + "epoch": 0.3076725025746653, + "grad_norm": 576.0, + "learning_rate": 3.4616374871266733e-07, + "logits/chosen": -2.5093750953674316, + "logits/rejected": -2.151562452316284, + "logps/chosen": -259.6000061035156, + "logps/rejected": -279.1000061035156, + "loss": 0.5195, + "rewards/accuracies": 0.7871862649917603, + "rewards/chosen": -0.30742186307907104, + "rewards/margins": 0.878125011920929, + "rewards/rejected": -1.183203101158142, + "step": 1195 + }, + { + "epoch": 0.30895983522142123, + "grad_norm": 476.0, + "learning_rate": 3.455200823892894e-07, + "logits/chosen": -2.659374952316284, + "logits/rejected": -2.4609375, + "logps/chosen": -337.20001220703125, + "logps/rejected": -336.79998779296875, + "loss": 0.4848, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.5731445550918579, + "rewards/margins": 1.0361328125, + "rewards/rejected": -1.607812523841858, + "step": 1200 + }, + { + "epoch": 0.31024716786817713, + "grad_norm": 532.0, + "learning_rate": 3.448764160659114e-07, + "logits/chosen": -2.7562499046325684, + "logits/rejected": -2.778125047683716, + "logps/chosen": -340.0, + "logps/rejected": -417.20001220703125, + "loss": 0.4984, + "rewards/accuracies": 0.7254762649536133, + "rewards/chosen": -0.7835937738418579, + "rewards/margins": 0.8890625238418579, + "rewards/rejected": -1.673437476158142, + "step": 1205 + }, + { + "epoch": 0.31153450051493303, + "grad_norm": 560.0, + "learning_rate": 3.4423274974253346e-07, + "logits/chosen": -2.765625, + "logits/rejected": -2.890625, + "logps/chosen": -366.3999938964844, + "logps/rejected": -357.20001220703125, + "loss": 0.482, + "rewards/accuracies": 0.7647727131843567, + "rewards/chosen": -0.27685546875, + "rewards/margins": 0.925000011920929, + "rewards/rejected": -1.2039062976837158, + "step": 1210 + }, + { + "epoch": 0.312821833161689, + "grad_norm": 362.0, + "learning_rate": 3.435890834191555e-07, + "logits/chosen": -2.5999999046325684, + "logits/rejected": -2.596874952316284, + "logps/chosen": -322.0, + "logps/rejected": -347.6000061035156, + "loss": 0.4703, + "rewards/accuracies": 0.7392308115959167, + "rewards/chosen": -0.603320300579071, + "rewards/margins": 1.092187523841858, + "rewards/rejected": -1.696874976158142, + "step": 1215 + }, + { + "epoch": 0.3141091658084449, + "grad_norm": 334.0, + "learning_rate": 3.429454170957775e-07, + "logits/chosen": -2.5687499046325684, + "logits/rejected": -2.5687499046325684, + "logps/chosen": -350.0, + "logps/rejected": -459.6000061035156, + "loss": 0.3215, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -0.9867187738418579, + "rewards/margins": 1.6062500476837158, + "rewards/rejected": -2.5953125953674316, + "step": 1220 + }, + { + "epoch": 0.31539649845520085, + "grad_norm": 628.0, + "learning_rate": 3.423017507723996e-07, + "logits/chosen": -2.549999952316284, + "logits/rejected": -2.734375, + "logps/chosen": -284.0, + "logps/rejected": -367.79998779296875, + "loss": 0.4773, + "rewards/accuracies": 0.701785683631897, + "rewards/chosen": -0.47050780057907104, + "rewards/margins": 1.09765625, + "rewards/rejected": -1.5703125, + "step": 1225 + }, + { + "epoch": 0.31668383110195675, + "grad_norm": 684.0, + "learning_rate": 3.4165808444902163e-07, + "logits/chosen": -2.4749999046325684, + "logits/rejected": -2.471874952316284, + "logps/chosen": -329.79998779296875, + "logps/rejected": -378.20001220703125, + "loss": 0.4934, + "rewards/accuracies": 0.7058441042900085, + "rewards/chosen": -0.8167968988418579, + "rewards/margins": 1.012109398841858, + "rewards/rejected": -1.826562523841858, + "step": 1230 + }, + { + "epoch": 0.31797116374871265, + "grad_norm": 596.0, + "learning_rate": 3.410144181256436e-07, + "logits/chosen": -2.2796874046325684, + "logits/rejected": -2.1953125, + "logps/chosen": -334.6000061035156, + "logps/rejected": -317.20001220703125, + "loss": 0.6461, + "rewards/accuracies": 0.6322552561759949, + "rewards/chosen": -0.815625011920929, + "rewards/margins": 0.670117199420929, + "rewards/rejected": -1.4880859851837158, + "step": 1235 + }, + { + "epoch": 0.3192584963954686, + "grad_norm": 916.0, + "learning_rate": 3.403707518022657e-07, + "logits/chosen": -2.637500047683716, + "logits/rejected": -2.7750000953674316, + "logps/chosen": -304.6000061035156, + "logps/rejected": -325.79998779296875, + "loss": 0.5703, + "rewards/accuracies": 0.6901924014091492, + "rewards/chosen": -0.66015625, + "rewards/margins": 0.9154297113418579, + "rewards/rejected": -1.5750000476837158, + "step": 1240 + }, + { + "epoch": 0.3205458290422245, + "grad_norm": 520.0, + "learning_rate": 3.3972708547888776e-07, + "logits/chosen": -2.596874952316284, + "logits/rejected": -2.653125047683716, + "logps/chosen": -301.8999938964844, + "logps/rejected": -294.70001220703125, + "loss": 0.5633, + "rewards/accuracies": 0.6676190495491028, + "rewards/chosen": -0.751904308795929, + "rewards/margins": 0.815625011920929, + "rewards/rejected": -1.5671875476837158, + "step": 1245 + }, + { + "epoch": 0.3218331616889804, + "grad_norm": 434.0, + "learning_rate": 3.3908341915550975e-07, + "logits/chosen": -2.6968750953674316, + "logits/rejected": -2.637500047683716, + "logps/chosen": -311.3999938964844, + "logps/rejected": -363.20001220703125, + "loss": 0.4941, + "rewards/accuracies": 0.705833375453949, + "rewards/chosen": -0.35087889432907104, + "rewards/margins": 1.037109375, + "rewards/rejected": -1.3898437023162842, + "step": 1250 + }, + { + "epoch": 0.32312049433573636, + "grad_norm": 624.0, + "learning_rate": 3.3843975283213184e-07, + "logits/chosen": -2.6781249046325684, + "logits/rejected": -2.5531249046325684, + "logps/chosen": -322.3999938964844, + "logps/rejected": -390.3999938964844, + "loss": 0.5656, + "rewards/accuracies": 0.6280769109725952, + "rewards/chosen": -0.3939453065395355, + "rewards/margins": 0.76171875, + "rewards/rejected": -1.153906226158142, + "step": 1255 + }, + { + "epoch": 0.32440782698249226, + "grad_norm": 520.0, + "learning_rate": 3.3779608650875383e-07, + "logits/chosen": -2.671875, + "logits/rejected": -2.78125, + "logps/chosen": -301.79998779296875, + "logps/rejected": -285.1000061035156, + "loss": 0.607, + "rewards/accuracies": 0.6060256361961365, + "rewards/chosen": -0.619384765625, + "rewards/margins": 0.561718761920929, + "rewards/rejected": -1.1828124523162842, + "step": 1260 + }, + { + "epoch": 0.3256951596292482, + "grad_norm": 600.0, + "learning_rate": 3.371524201853759e-07, + "logits/chosen": -2.706249952316284, + "logits/rejected": -2.71875, + "logps/chosen": -253.0, + "logps/rejected": -276.79998779296875, + "loss": 0.4697, + "rewards/accuracies": 0.6843590140342712, + "rewards/chosen": -0.569042980670929, + "rewards/margins": 0.9945312738418579, + "rewards/rejected": -1.5656249523162842, + "step": 1265 + }, + { + "epoch": 0.3269824922760041, + "grad_norm": 652.0, + "learning_rate": 3.3650875386199797e-07, + "logits/chosen": -2.6624999046325684, + "logits/rejected": -2.653125047683716, + "logps/chosen": -317.20001220703125, + "logps/rejected": -361.6000061035156, + "loss": 0.5469, + "rewards/accuracies": 0.6698077917098999, + "rewards/chosen": -0.8203125, + "rewards/margins": 0.8280273675918579, + "rewards/rejected": -1.646093726158142, + "step": 1270 + }, + { + "epoch": 0.32826982492276, + "grad_norm": 374.0, + "learning_rate": 3.3586508753861996e-07, + "logits/chosen": -2.8375000953674316, + "logits/rejected": -2.828125, + "logps/chosen": -369.3999938964844, + "logps/rejected": -355.20001220703125, + "loss": 0.5504, + "rewards/accuracies": 0.6591483950614929, + "rewards/chosen": -0.33906251192092896, + "rewards/margins": 0.6136718988418579, + "rewards/rejected": -0.953906238079071, + "step": 1275 + }, + { + "epoch": 0.329557157569516, + "grad_norm": 744.0, + "learning_rate": 3.35221421215242e-07, + "logits/chosen": -2.6343750953674316, + "logits/rejected": -2.6968750953674316, + "logps/chosen": -373.6000061035156, + "logps/rejected": -429.6000061035156, + "loss": 0.4781, + "rewards/accuracies": 0.8089286088943481, + "rewards/chosen": -0.6451171636581421, + "rewards/margins": 0.9945312738418579, + "rewards/rejected": -1.638671875, + "step": 1280 + }, + { + "epoch": 0.3308444902162719, + "grad_norm": 772.0, + "learning_rate": 3.34577754891864e-07, + "logits/chosen": -2.390625, + "logits/rejected": -2.862499952316284, + "logps/chosen": -311.0, + "logps/rejected": -259.3999938964844, + "loss": 0.5742, + "rewards/accuracies": 0.6872727274894714, + "rewards/chosen": -0.4505859315395355, + "rewards/margins": 0.958984375, + "rewards/rejected": -1.411718726158142, + "step": 1285 + }, + { + "epoch": 0.3321318228630278, + "grad_norm": 484.0, + "learning_rate": 3.339340885684861e-07, + "logits/chosen": -2.4312500953674316, + "logits/rejected": -2.4437499046325684, + "logps/chosen": -312.79998779296875, + "logps/rejected": -394.20001220703125, + "loss": 0.4277, + "rewards/accuracies": 0.7455769777297974, + "rewards/chosen": -0.838671863079071, + "rewards/margins": 1.1843750476837158, + "rewards/rejected": -2.0218749046325684, + "step": 1290 + }, + { + "epoch": 0.33341915550978374, + "grad_norm": 572.0, + "learning_rate": 3.3329042224510813e-07, + "logits/chosen": -2.575000047683716, + "logits/rejected": -2.5625, + "logps/chosen": -336.3999938964844, + "logps/rejected": -434.3999938964844, + "loss": 0.4305, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.921093761920929, + "rewards/margins": 1.228906273841858, + "rewards/rejected": -2.1500000953674316, + "step": 1295 + }, + { + "epoch": 0.33470648815653964, + "grad_norm": 604.0, + "learning_rate": 3.326467559217301e-07, + "logits/chosen": -2.5250000953674316, + "logits/rejected": -2.5625, + "logps/chosen": -384.0, + "logps/rejected": -440.0, + "loss": 0.4246, + "rewards/accuracies": 0.8270238637924194, + "rewards/chosen": -0.5167480707168579, + "rewards/margins": 1.1589844226837158, + "rewards/rejected": -1.678125023841858, + "step": 1300 + }, + { + "epoch": 0.3359938208032956, + "grad_norm": 348.0, + "learning_rate": 3.320030895983522e-07, + "logits/chosen": -2.4437499046325684, + "logits/rejected": -2.612499952316284, + "logps/chosen": -271.0, + "logps/rejected": -328.0, + "loss": 0.434, + "rewards/accuracies": 0.7549999952316284, + "rewards/chosen": -0.6009765863418579, + "rewards/margins": 1.3292968273162842, + "rewards/rejected": -1.9296875, + "step": 1305 + }, + { + "epoch": 0.3372811534500515, + "grad_norm": 648.0, + "learning_rate": 3.3135942327497426e-07, + "logits/chosen": -2.3531250953674316, + "logits/rejected": -2.0921874046325684, + "logps/chosen": -283.6000061035156, + "logps/rejected": -354.3999938964844, + "loss": 0.4793, + "rewards/accuracies": 0.7186111211776733, + "rewards/chosen": -0.46074217557907104, + "rewards/margins": 1.0402343273162842, + "rewards/rejected": -1.5014159679412842, + "step": 1310 + }, + { + "epoch": 0.3385684860968074, + "grad_norm": 620.0, + "learning_rate": 3.3071575695159625e-07, + "logits/chosen": -2.378124952316284, + "logits/rejected": -2.6109375953674316, + "logps/chosen": -314.20001220703125, + "logps/rejected": -343.8999938964844, + "loss": 0.5008, + "rewards/accuracies": 0.7196428775787354, + "rewards/chosen": -0.571044921875, + "rewards/margins": 1.2423827648162842, + "rewards/rejected": -1.814453125, + "step": 1315 + }, + { + "epoch": 0.33985581874356335, + "grad_norm": 596.0, + "learning_rate": 3.3007209062821835e-07, + "logits/chosen": -2.671875, + "logits/rejected": -2.65625, + "logps/chosen": -307.6000061035156, + "logps/rejected": -402.6000061035156, + "loss": 0.5398, + "rewards/accuracies": 0.7396212220191956, + "rewards/chosen": -0.617968738079071, + "rewards/margins": 0.884765625, + "rewards/rejected": -1.5031249523162842, + "step": 1320 + }, + { + "epoch": 0.34114315139031925, + "grad_norm": 676.0, + "learning_rate": 3.2942842430484033e-07, + "logits/chosen": -2.5625, + "logits/rejected": -2.65625, + "logps/chosen": -353.79998779296875, + "logps/rejected": -313.0, + "loss": 0.5391, + "rewards/accuracies": 0.6944047808647156, + "rewards/chosen": -0.546093761920929, + "rewards/margins": 0.746874988079071, + "rewards/rejected": -1.29296875, + "step": 1325 + }, + { + "epoch": 0.34243048403707516, + "grad_norm": 476.0, + "learning_rate": 3.287847579814624e-07, + "logits/chosen": -2.242968797683716, + "logits/rejected": -2.674999952316284, + "logps/chosen": -285.8999938964844, + "logps/rejected": -316.3999938964844, + "loss": 0.5215, + "rewards/accuracies": 0.7234890460968018, + "rewards/chosen": -0.2855468690395355, + "rewards/margins": 0.862500011920929, + "rewards/rejected": -1.1492187976837158, + "step": 1330 + }, + { + "epoch": 0.3437178166838311, + "grad_norm": 576.0, + "learning_rate": 3.2814109165808447e-07, + "logits/chosen": -2.5218749046325684, + "logits/rejected": -2.578125, + "logps/chosen": -303.3999938964844, + "logps/rejected": -354.6000061035156, + "loss": 0.5285, + "rewards/accuracies": 0.6717424392700195, + "rewards/chosen": -0.746874988079071, + "rewards/margins": 0.875244140625, + "rewards/rejected": -1.6218750476837158, + "step": 1335 + }, + { + "epoch": 0.345005149330587, + "grad_norm": 376.0, + "learning_rate": 3.2749742533470646e-07, + "logits/chosen": -2.809375047683716, + "logits/rejected": -2.846874952316284, + "logps/chosen": -284.20001220703125, + "logps/rejected": -306.0, + "loss": 0.4848, + "rewards/accuracies": 0.7240909337997437, + "rewards/chosen": 0.010937499813735485, + "rewards/margins": 0.7476562261581421, + "rewards/rejected": -0.7378906011581421, + "step": 1340 + }, + { + "epoch": 0.34629248197734297, + "grad_norm": 700.0, + "learning_rate": 3.268537590113285e-07, + "logits/chosen": -2.65625, + "logits/rejected": -2.5625, + "logps/chosen": -256.3999938964844, + "logps/rejected": -298.20001220703125, + "loss": 0.5699, + "rewards/accuracies": 0.6326282024383545, + "rewards/chosen": -0.41425782442092896, + "rewards/margins": 0.6494140625, + "rewards/rejected": -1.062890648841858, + "step": 1345 + }, + { + "epoch": 0.34757981462409887, + "grad_norm": 668.0, + "learning_rate": 3.262100926879506e-07, + "logits/chosen": -2.456249952316284, + "logits/rejected": -2.3421874046325684, + "logps/chosen": -318.0, + "logps/rejected": -344.0, + "loss": 0.6129, + "rewards/accuracies": 0.6523810029029846, + "rewards/chosen": -0.40791016817092896, + "rewards/margins": 0.5611327886581421, + "rewards/rejected": -0.9664062261581421, + "step": 1350 + }, + { + "epoch": 0.34886714727085477, + "grad_norm": 540.0, + "learning_rate": 3.255664263645726e-07, + "logits/chosen": -2.59375, + "logits/rejected": -2.578125, + "logps/chosen": -297.6000061035156, + "logps/rejected": -391.20001220703125, + "loss": 0.4359, + "rewards/accuracies": 0.7389286160469055, + "rewards/chosen": -0.796875, + "rewards/margins": 1.446874976158142, + "rewards/rejected": -2.2421875, + "step": 1355 + }, + { + "epoch": 0.35015447991761073, + "grad_norm": 536.0, + "learning_rate": 3.2492276004119463e-07, + "logits/chosen": -2.621875047683716, + "logits/rejected": -2.5999999046325684, + "logps/chosen": -285.0, + "logps/rejected": -326.0, + "loss": 0.5176, + "rewards/accuracies": 0.7292748689651489, + "rewards/chosen": -0.618359386920929, + "rewards/margins": 1.012109398841858, + "rewards/rejected": -1.62890625, + "step": 1360 + }, + { + "epoch": 0.35144181256436663, + "grad_norm": 330.0, + "learning_rate": 3.242790937178166e-07, + "logits/chosen": -2.7562499046325684, + "logits/rejected": -2.8125, + "logps/chosen": -328.20001220703125, + "logps/rejected": -404.79998779296875, + "loss": 0.527, + "rewards/accuracies": 0.7211538553237915, + "rewards/chosen": -0.5657714605331421, + "rewards/margins": 0.8179687261581421, + "rewards/rejected": -1.3820312023162842, + "step": 1365 + }, + { + "epoch": 0.35272914521112253, + "grad_norm": 740.0, + "learning_rate": 3.236354273944387e-07, + "logits/chosen": -2.6156249046325684, + "logits/rejected": -2.512500047683716, + "logps/chosen": -289.0, + "logps/rejected": -276.79998779296875, + "loss": 0.6602, + "rewards/accuracies": 0.5599359273910522, + "rewards/chosen": -0.8765624761581421, + "rewards/margins": 0.461181640625, + "rewards/rejected": -1.3367187976837158, + "step": 1370 + }, + { + "epoch": 0.3540164778578785, + "grad_norm": 520.0, + "learning_rate": 3.2299176107106076e-07, + "logits/chosen": -2.6781249046325684, + "logits/rejected": -2.684375047683716, + "logps/chosen": -347.6000061035156, + "logps/rejected": -440.3999938964844, + "loss": 0.6094, + "rewards/accuracies": 0.59333336353302, + "rewards/chosen": -0.6468750238418579, + "rewards/margins": 0.563671886920929, + "rewards/rejected": -1.2109375, + "step": 1375 + }, + { + "epoch": 0.3553038105046344, + "grad_norm": 416.0, + "learning_rate": 3.2234809474768275e-07, + "logits/chosen": -2.75, + "logits/rejected": -2.799999952316284, + "logps/chosen": -369.20001220703125, + "logps/rejected": -365.6000061035156, + "loss": 0.527, + "rewards/accuracies": 0.7844414114952087, + "rewards/chosen": -0.727734386920929, + "rewards/margins": 0.7914062738418579, + "rewards/rejected": -1.517968773841858, + "step": 1380 + }, + { + "epoch": 0.35659114315139034, + "grad_norm": 620.0, + "learning_rate": 3.2170442842430485e-07, + "logits/chosen": -2.6312499046325684, + "logits/rejected": -2.590625047683716, + "logps/chosen": -253.60000610351562, + "logps/rejected": -313.6000061035156, + "loss": 0.5414, + "rewards/accuracies": 0.6357900500297546, + "rewards/chosen": -0.3921875059604645, + "rewards/margins": 0.665820300579071, + "rewards/rejected": -1.0575439929962158, + "step": 1385 + }, + { + "epoch": 0.35787847579814624, + "grad_norm": 388.0, + "learning_rate": 3.2106076210092684e-07, + "logits/chosen": -2.625, + "logits/rejected": -2.700000047683716, + "logps/chosen": -359.79998779296875, + "logps/rejected": -385.6000061035156, + "loss": 0.4742, + "rewards/accuracies": 0.7514393925666809, + "rewards/chosen": -0.612500011920929, + "rewards/margins": 0.964062511920929, + "rewards/rejected": -1.575781226158142, + "step": 1390 + }, + { + "epoch": 0.35916580844490215, + "grad_norm": 474.0, + "learning_rate": 3.204170957775489e-07, + "logits/chosen": -2.4281249046325684, + "logits/rejected": -2.2640624046325684, + "logps/chosen": -247.8000030517578, + "logps/rejected": -292.3999938964844, + "loss": 0.4707, + "rewards/accuracies": 0.6819047927856445, + "rewards/chosen": -0.4749999940395355, + "rewards/margins": 1.0242187976837158, + "rewards/rejected": -1.497656226158142, + "step": 1395 + }, + { + "epoch": 0.3604531410916581, + "grad_norm": 344.0, + "learning_rate": 3.19773429454171e-07, + "logits/chosen": -2.7125000953674316, + "logits/rejected": -2.7093749046325684, + "logps/chosen": -362.3999938964844, + "logps/rejected": -363.20001220703125, + "loss": 0.4832, + "rewards/accuracies": 0.7291666865348816, + "rewards/chosen": -0.609570324420929, + "rewards/margins": 1.036718726158142, + "rewards/rejected": -1.64453125, + "step": 1400 + }, + { + "epoch": 0.361740473738414, + "grad_norm": 624.0, + "learning_rate": 3.1912976313079296e-07, + "logits/chosen": -2.734375, + "logits/rejected": -2.746875047683716, + "logps/chosen": -313.6000061035156, + "logps/rejected": -306.0, + "loss": 0.6168, + "rewards/accuracies": 0.6620146632194519, + "rewards/chosen": -0.09121093899011612, + "rewards/margins": 0.5165771245956421, + "rewards/rejected": -0.607714831829071, + "step": 1405 + }, + { + "epoch": 0.3630278063851699, + "grad_norm": 820.0, + "learning_rate": 3.18486096807415e-07, + "logits/chosen": -2.512500047683716, + "logits/rejected": -2.3265624046325684, + "logps/chosen": -305.79998779296875, + "logps/rejected": -410.0, + "loss": 0.4492, + "rewards/accuracies": 0.8063095211982727, + "rewards/chosen": -0.708984375, + "rewards/margins": 1.0636718273162842, + "rewards/rejected": -1.7734375, + "step": 1410 + }, + { + "epoch": 0.36431513903192586, + "grad_norm": 416.0, + "learning_rate": 3.178424304840371e-07, + "logits/chosen": -2.59375, + "logits/rejected": -2.575000047683716, + "logps/chosen": -309.6000061035156, + "logps/rejected": -349.6000061035156, + "loss": 0.4888, + "rewards/accuracies": 0.7802197933197021, + "rewards/chosen": -0.61279296875, + "rewards/margins": 1.162500023841858, + "rewards/rejected": -1.775976538658142, + "step": 1415 + }, + { + "epoch": 0.36560247167868176, + "grad_norm": 422.0, + "learning_rate": 3.171987641606591e-07, + "logits/chosen": -2.390625, + "logits/rejected": -2.503124952316284, + "logps/chosen": -313.29998779296875, + "logps/rejected": -348.20001220703125, + "loss": 0.541, + "rewards/accuracies": 0.6946429014205933, + "rewards/chosen": -0.875, + "rewards/margins": 0.866992175579071, + "rewards/rejected": -1.742578148841858, + "step": 1420 + }, + { + "epoch": 0.3668898043254377, + "grad_norm": 364.0, + "learning_rate": 3.1655509783728114e-07, + "logits/chosen": -2.734375, + "logits/rejected": -2.7906250953674316, + "logps/chosen": -347.6000061035156, + "logps/rejected": -410.3999938964844, + "loss": 0.509, + "rewards/accuracies": 0.724682629108429, + "rewards/chosen": -0.5784698724746704, + "rewards/margins": 0.919921875, + "rewards/rejected": -1.494531273841858, + "step": 1425 + }, + { + "epoch": 0.3681771369721936, + "grad_norm": 400.0, + "learning_rate": 3.159114315139032e-07, + "logits/chosen": -2.628124952316284, + "logits/rejected": -2.7249999046325684, + "logps/chosen": -280.3999938964844, + "logps/rejected": -244.1999969482422, + "loss": 0.5391, + "rewards/accuracies": 0.6097726821899414, + "rewards/chosen": -0.4898437559604645, + "rewards/margins": 0.632031261920929, + "rewards/rejected": -1.1203124523162842, + "step": 1430 + }, + { + "epoch": 0.3694644696189495, + "grad_norm": 444.0, + "learning_rate": 3.152677651905252e-07, + "logits/chosen": -2.190624952316284, + "logits/rejected": -2.46875, + "logps/chosen": -272.20001220703125, + "logps/rejected": -342.0, + "loss": 0.3564, + "rewards/accuracies": 0.8623737096786499, + "rewards/chosen": -0.76123046875, + "rewards/margins": 1.3914062976837158, + "rewards/rejected": -2.147656202316284, + "step": 1435 + }, + { + "epoch": 0.3707518022657055, + "grad_norm": 1152.0, + "learning_rate": 3.1462409886714726e-07, + "logits/chosen": -2.543750047683716, + "logits/rejected": -2.4437499046325684, + "logps/chosen": -311.70001220703125, + "logps/rejected": -324.8999938964844, + "loss": 0.5576, + "rewards/accuracies": 0.7600000500679016, + "rewards/chosen": -0.842578113079071, + "rewards/margins": 0.8238281011581421, + "rewards/rejected": -1.665624976158142, + "step": 1440 + }, + { + "epoch": 0.3720391349124614, + "grad_norm": 624.0, + "learning_rate": 3.1398043254376925e-07, + "logits/chosen": -2.581249952316284, + "logits/rejected": -2.674999952316284, + "logps/chosen": -341.79998779296875, + "logps/rejected": -370.79998779296875, + "loss": 0.4848, + "rewards/accuracies": 0.7094047665596008, + "rewards/chosen": -0.6126953363418579, + "rewards/margins": 0.9398437738418579, + "rewards/rejected": -1.5515625476837158, + "step": 1445 + }, + { + "epoch": 0.3733264675592173, + "grad_norm": 344.0, + "learning_rate": 3.1333676622039135e-07, + "logits/chosen": -2.840625047683716, + "logits/rejected": -2.871875047683716, + "logps/chosen": -350.79998779296875, + "logps/rejected": -322.3999938964844, + "loss": 0.5797, + "rewards/accuracies": 0.6589743494987488, + "rewards/chosen": -0.2582031190395355, + "rewards/margins": 0.5865234136581421, + "rewards/rejected": -0.844531238079071, + "step": 1450 + }, + { + "epoch": 0.37461380020597324, + "grad_norm": 664.0, + "learning_rate": 3.126930998970134e-07, + "logits/chosen": -2.6656250953674316, + "logits/rejected": -2.546875, + "logps/chosen": -315.6000061035156, + "logps/rejected": -367.20001220703125, + "loss": 0.4727, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.903515636920929, + "rewards/margins": 0.971484363079071, + "rewards/rejected": -1.875, + "step": 1455 + }, + { + "epoch": 0.37590113285272914, + "grad_norm": 544.0, + "learning_rate": 3.120494335736354e-07, + "logits/chosen": -2.515625, + "logits/rejected": -2.690624952316284, + "logps/chosen": -329.0, + "logps/rejected": -356.3999938964844, + "loss": 0.5746, + "rewards/accuracies": 0.6949999928474426, + "rewards/chosen": -0.564868152141571, + "rewards/margins": 0.668896496295929, + "rewards/rejected": -1.232031226158142, + "step": 1460 + }, + { + "epoch": 0.3771884654994851, + "grad_norm": 466.0, + "learning_rate": 3.114057672502575e-07, + "logits/chosen": -2.65625, + "logits/rejected": -2.640625, + "logps/chosen": -333.20001220703125, + "logps/rejected": -366.3999938964844, + "loss": 0.4285, + "rewards/accuracies": 0.8238420486450195, + "rewards/chosen": -0.4585937559604645, + "rewards/margins": 1.157812476158142, + "rewards/rejected": -1.618749976158142, + "step": 1465 + }, + { + "epoch": 0.378475798146241, + "grad_norm": 584.0, + "learning_rate": 3.1076210092687947e-07, + "logits/chosen": -2.671875, + "logits/rejected": -2.6500000953674316, + "logps/chosen": -314.79998779296875, + "logps/rejected": -304.20001220703125, + "loss": 0.498, + "rewards/accuracies": 0.716864824295044, + "rewards/chosen": -0.22148437798023224, + "rewards/margins": 0.960156261920929, + "rewards/rejected": -1.181640625, + "step": 1470 + }, + { + "epoch": 0.3797631307929969, + "grad_norm": 576.0, + "learning_rate": 3.101184346035015e-07, + "logits/chosen": -2.753124952316284, + "logits/rejected": -2.8187499046325684, + "logps/chosen": -342.0, + "logps/rejected": -387.0, + "loss": 0.507, + "rewards/accuracies": 0.7564102411270142, + "rewards/chosen": -0.6007324457168579, + "rewards/margins": 0.828906238079071, + "rewards/rejected": -1.4269530773162842, + "step": 1475 + }, + { + "epoch": 0.38105046343975285, + "grad_norm": 2672.0, + "learning_rate": 3.094747682801236e-07, + "logits/chosen": -2.6875, + "logits/rejected": -2.0227417945861816, + "logps/chosen": -224.85000610351562, + "logps/rejected": -298.70001220703125, + "loss": 0.5695, + "rewards/accuracies": 0.6308333277702332, + "rewards/chosen": -0.416015625, + "rewards/margins": 0.6177734136581421, + "rewards/rejected": -1.0321776866912842, + "step": 1480 + }, + { + "epoch": 0.38233779608650875, + "grad_norm": 360.0, + "learning_rate": 3.088311019567456e-07, + "logits/chosen": -2.596874952316284, + "logits/rejected": -2.59375, + "logps/chosen": -334.0, + "logps/rejected": -367.20001220703125, + "loss": 0.3908, + "rewards/accuracies": 0.8008116483688354, + "rewards/chosen": -0.4423584043979645, + "rewards/margins": 1.267968773841858, + "rewards/rejected": -1.7101562023162842, + "step": 1485 + }, + { + "epoch": 0.38362512873326465, + "grad_norm": 436.0, + "learning_rate": 3.0818743563336764e-07, + "logits/chosen": -2.6875, + "logits/rejected": -2.596874952316284, + "logps/chosen": -329.20001220703125, + "logps/rejected": -335.20001220703125, + "loss": 0.4805, + "rewards/accuracies": 0.7127564549446106, + "rewards/chosen": -0.41093748807907104, + "rewards/margins": 0.914843738079071, + "rewards/rejected": -1.325781226158142, + "step": 1490 + }, + { + "epoch": 0.3849124613800206, + "grad_norm": 368.0, + "learning_rate": 3.0754376930998973e-07, + "logits/chosen": -2.762500047683716, + "logits/rejected": -2.75, + "logps/chosen": -310.0, + "logps/rejected": -311.79998779296875, + "loss": 0.5703, + "rewards/accuracies": 0.6324999928474426, + "rewards/chosen": -0.3408203125, + "rewards/margins": 0.6650390625, + "rewards/rejected": -1.0070312023162842, + "step": 1495 + }, + { + "epoch": 0.3861997940267765, + "grad_norm": 358.0, + "learning_rate": 3.069001029866117e-07, + "logits/chosen": -2.590625047683716, + "logits/rejected": -2.674999952316284, + "logps/chosen": -307.20001220703125, + "logps/rejected": -355.20001220703125, + "loss": 0.5371, + "rewards/accuracies": 0.7116667032241821, + "rewards/chosen": -0.572802722454071, + "rewards/margins": 0.962207019329071, + "rewards/rejected": -1.537500023841858, + "step": 1500 + }, + { + "epoch": 0.38748712667353247, + "grad_norm": 492.0, + "learning_rate": 3.0625643666323377e-07, + "logits/chosen": -2.6937499046325684, + "logits/rejected": -2.762500047683716, + "logps/chosen": -336.0, + "logps/rejected": -398.3999938964844, + "loss": 0.5687, + "rewards/accuracies": 0.7165476083755493, + "rewards/chosen": -0.615283191204071, + "rewards/margins": 0.748242199420929, + "rewards/rejected": -1.364843726158142, + "step": 1505 + }, + { + "epoch": 0.38877445932028837, + "grad_norm": 728.0, + "learning_rate": 3.056127703398558e-07, + "logits/chosen": -2.671875, + "logits/rejected": -2.5140624046325684, + "logps/chosen": -337.79998779296875, + "logps/rejected": -295.20001220703125, + "loss": 0.5938, + "rewards/accuracies": 0.6627289652824402, + "rewards/chosen": -0.5433593988418579, + "rewards/margins": 0.48828125, + "rewards/rejected": -1.0304687023162842, + "step": 1510 + }, + { + "epoch": 0.39006179196704427, + "grad_norm": 402.0, + "learning_rate": 3.0496910401647785e-07, + "logits/chosen": -2.8218750953674316, + "logits/rejected": -2.5999999046325684, + "logps/chosen": -329.6000061035156, + "logps/rejected": -347.20001220703125, + "loss": 0.5906, + "rewards/accuracies": 0.7293182015419006, + "rewards/chosen": -0.5601562261581421, + "rewards/margins": 0.7466796636581421, + "rewards/rejected": -1.3093750476837158, + "step": 1515 + }, + { + "epoch": 0.3913491246138002, + "grad_norm": 696.0, + "learning_rate": 3.043254376930999e-07, + "logits/chosen": -2.5875000953674316, + "logits/rejected": -2.5, + "logps/chosen": -333.0, + "logps/rejected": -432.3999938964844, + "loss": 0.4361, + "rewards/accuracies": 0.8241666555404663, + "rewards/chosen": -0.8125, + "rewards/margins": 1.228906273841858, + "rewards/rejected": -2.0374999046325684, + "step": 1520 + }, + { + "epoch": 0.3926364572605561, + "grad_norm": 528.0, + "learning_rate": 3.036817713697219e-07, + "logits/chosen": -2.621875047683716, + "logits/rejected": -2.671875, + "logps/chosen": -264.20001220703125, + "logps/rejected": -266.3999938964844, + "loss": 0.5043, + "rewards/accuracies": 0.6777622103691101, + "rewards/chosen": -0.24155274033546448, + "rewards/margins": 0.755078136920929, + "rewards/rejected": -0.999218761920929, + "step": 1525 + }, + { + "epoch": 0.393923789907312, + "grad_norm": 524.0, + "learning_rate": 3.03038105046344e-07, + "logits/chosen": -2.487499952316284, + "logits/rejected": -2.5718750953674316, + "logps/chosen": -280.5, + "logps/rejected": -354.29998779296875, + "loss": 0.5207, + "rewards/accuracies": 0.7708333730697632, + "rewards/chosen": -0.5279296636581421, + "rewards/margins": 0.940234363079071, + "rewards/rejected": -1.467187523841858, + "step": 1530 + }, + { + "epoch": 0.395211122554068, + "grad_norm": 956.0, + "learning_rate": 3.0239443872296597e-07, + "logits/chosen": -2.5843749046325684, + "logits/rejected": -2.6468749046325684, + "logps/chosen": -297.6000061035156, + "logps/rejected": -337.3999938964844, + "loss": 0.5137, + "rewards/accuracies": 0.7225000262260437, + "rewards/chosen": -0.80078125, + "rewards/margins": 0.9085937738418579, + "rewards/rejected": -1.7101562023162842, + "step": 1535 + }, + { + "epoch": 0.3964984552008239, + "grad_norm": 532.0, + "learning_rate": 3.01750772399588e-07, + "logits/chosen": -2.515625, + "logits/rejected": -2.596874952316284, + "logps/chosen": -346.3999938964844, + "logps/rejected": -406.3999938964844, + "loss": 0.5328, + "rewards/accuracies": 0.6608333587646484, + "rewards/chosen": -0.601855456829071, + "rewards/margins": 0.9140625, + "rewards/rejected": -1.515625, + "step": 1540 + }, + { + "epoch": 0.39778578784757984, + "grad_norm": 414.0, + "learning_rate": 3.011071060762101e-07, + "logits/chosen": -2.3671875, + "logits/rejected": -2.4546875953674316, + "logps/chosen": -273.6000061035156, + "logps/rejected": -356.0, + "loss": 0.4031, + "rewards/accuracies": 0.7854545712471008, + "rewards/chosen": -0.3343261778354645, + "rewards/margins": 1.1648437976837158, + "rewards/rejected": -1.501562476158142, + "step": 1545 + }, + { + "epoch": 0.39907312049433574, + "grad_norm": 708.0, + "learning_rate": 3.004634397528321e-07, + "logits/chosen": -2.5843749046325684, + "logits/rejected": -2.614062547683716, + "logps/chosen": -300.6000061035156, + "logps/rejected": -288.5, + "loss": 0.5898, + "rewards/accuracies": 0.6898809671401978, + "rewards/chosen": -0.6939452886581421, + "rewards/margins": 0.696484386920929, + "rewards/rejected": -1.392578125, + "step": 1550 + }, + { + "epoch": 0.40036045314109164, + "grad_norm": 564.0, + "learning_rate": 2.9981977342945414e-07, + "logits/chosen": -2.612499952316284, + "logits/rejected": -2.674999952316284, + "logps/chosen": -328.0, + "logps/rejected": -376.0, + "loss": 0.5141, + "rewards/accuracies": 0.7025367021560669, + "rewards/chosen": -0.6408447027206421, + "rewards/margins": 0.859179675579071, + "rewards/rejected": -1.501562476158142, + "step": 1555 + }, + { + "epoch": 0.4016477857878476, + "grad_norm": 688.0, + "learning_rate": 2.9917610710607623e-07, + "logits/chosen": -2.668750047683716, + "logits/rejected": -2.643749952316284, + "logps/chosen": -374.0, + "logps/rejected": -371.6000061035156, + "loss": 0.4844, + "rewards/accuracies": 0.7467857599258423, + "rewards/chosen": -0.3626953065395355, + "rewards/margins": 0.875, + "rewards/rejected": -1.236328125, + "step": 1560 + }, + { + "epoch": 0.4029351184346035, + "grad_norm": 576.0, + "learning_rate": 2.985324407826982e-07, + "logits/chosen": -2.6812500953674316, + "logits/rejected": -2.6156249046325684, + "logps/chosen": -312.0, + "logps/rejected": -313.3999938964844, + "loss": 0.5473, + "rewards/accuracies": 0.6609523892402649, + "rewards/chosen": -0.37238770723342896, + "rewards/margins": 0.7494140863418579, + "rewards/rejected": -1.1203124523162842, + "step": 1565 + }, + { + "epoch": 0.4042224510813594, + "grad_norm": 438.0, + "learning_rate": 2.9788877445932027e-07, + "logits/chosen": -2.628124952316284, + "logits/rejected": -2.596874952316284, + "logps/chosen": -334.0, + "logps/rejected": -392.79998779296875, + "loss": 0.518, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.028906226158142, + "rewards/margins": 1.0, + "rewards/rejected": -2.03125, + "step": 1570 + }, + { + "epoch": 0.40550978372811536, + "grad_norm": 540.0, + "learning_rate": 2.972451081359423e-07, + "logits/chosen": -2.339062452316284, + "logits/rejected": -2.3187499046325684, + "logps/chosen": -304.6000061035156, + "logps/rejected": -397.20001220703125, + "loss": 0.4434, + "rewards/accuracies": 0.8075000643730164, + "rewards/chosen": -1.0773437023162842, + "rewards/margins": 1.517187476158142, + "rewards/rejected": -2.59375, + "step": 1575 + }, + { + "epoch": 0.40679711637487126, + "grad_norm": 480.0, + "learning_rate": 2.9660144181256435e-07, + "logits/chosen": -2.746875047683716, + "logits/rejected": -2.8062500953674316, + "logps/chosen": -264.0, + "logps/rejected": -232.1999969482422, + "loss": 0.5109, + "rewards/accuracies": 0.7144871950149536, + "rewards/chosen": -0.3717285096645355, + "rewards/margins": 0.713671863079071, + "rewards/rejected": -1.084375023841858, + "step": 1580 + }, + { + "epoch": 0.4080844490216272, + "grad_norm": 532.0, + "learning_rate": 2.959577754891864e-07, + "logits/chosen": -2.7750000953674316, + "logits/rejected": -2.7437500953674316, + "logps/chosen": -327.20001220703125, + "logps/rejected": -308.79998779296875, + "loss": 0.5535, + "rewards/accuracies": 0.6758333444595337, + "rewards/chosen": -0.28593748807907104, + "rewards/margins": 0.707812488079071, + "rewards/rejected": -0.995312511920929, + "step": 1585 + }, + { + "epoch": 0.4093717816683831, + "grad_norm": 462.0, + "learning_rate": 2.9531410916580844e-07, + "logits/chosen": -2.71875, + "logits/rejected": -2.596874952316284, + "logps/chosen": -282.6000061035156, + "logps/rejected": -331.79998779296875, + "loss": 0.4434, + "rewards/accuracies": 0.7734920382499695, + "rewards/chosen": -0.3519531190395355, + "rewards/margins": 1.0105469226837158, + "rewards/rejected": -1.3624999523162842, + "step": 1590 + }, + { + "epoch": 0.410659114315139, + "grad_norm": 520.0, + "learning_rate": 2.946704428424305e-07, + "logits/chosen": -2.731250047683716, + "logits/rejected": -2.534374952316284, + "logps/chosen": -310.0, + "logps/rejected": -320.20001220703125, + "loss": 0.4527, + "rewards/accuracies": 0.7508333921432495, + "rewards/chosen": -0.31694334745407104, + "rewards/margins": 0.967578113079071, + "rewards/rejected": -1.2824218273162842, + "step": 1595 + }, + { + "epoch": 0.411946446961895, + "grad_norm": 784.0, + "learning_rate": 2.940267765190525e-07, + "logits/chosen": -2.625, + "logits/rejected": -2.737499952316284, + "logps/chosen": -277.20001220703125, + "logps/rejected": -281.20001220703125, + "loss": 0.5211, + "rewards/accuracies": 0.6996794939041138, + "rewards/chosen": -0.4035400450229645, + "rewards/margins": 0.652148425579071, + "rewards/rejected": -1.053125023841858, + "step": 1600 + }, + { + "epoch": 0.4132337796086509, + "grad_norm": 412.0, + "learning_rate": 2.933831101956745e-07, + "logits/chosen": -2.5250000953674316, + "logits/rejected": -2.5250000953674316, + "logps/chosen": -297.0, + "logps/rejected": -295.20001220703125, + "loss": 0.5049, + "rewards/accuracies": 0.7454545497894287, + "rewards/chosen": -0.30351561307907104, + "rewards/margins": 0.861328125, + "rewards/rejected": -1.164404273033142, + "step": 1605 + }, + { + "epoch": 0.4145211122554068, + "grad_norm": 548.0, + "learning_rate": 2.927394438722966e-07, + "logits/chosen": -2.6500000953674316, + "logits/rejected": -2.6781249046325684, + "logps/chosen": -279.6000061035156, + "logps/rejected": -342.3999938964844, + "loss": 0.4543, + "rewards/accuracies": 0.7886905074119568, + "rewards/chosen": -0.513671875, + "rewards/margins": 1.066015601158142, + "rewards/rejected": -1.580468773841858, + "step": 1610 + }, + { + "epoch": 0.41580844490216273, + "grad_norm": 524.0, + "learning_rate": 2.920957775489186e-07, + "logits/chosen": -2.534374952316284, + "logits/rejected": -2.5250000953674316, + "logps/chosen": -331.20001220703125, + "logps/rejected": -366.3999938964844, + "loss": 0.5953, + "rewards/accuracies": 0.6791941523551941, + "rewards/chosen": -0.7906249761581421, + "rewards/margins": 0.6976562738418579, + "rewards/rejected": -1.4871094226837158, + "step": 1615 + }, + { + "epoch": 0.41709577754891863, + "grad_norm": 812.0, + "learning_rate": 2.9145211122554064e-07, + "logits/chosen": -2.799999952316284, + "logits/rejected": -2.606250047683716, + "logps/chosen": -322.0, + "logps/rejected": -304.6000061035156, + "loss": 0.6512, + "rewards/accuracies": 0.6438094973564148, + "rewards/chosen": -0.424224853515625, + "rewards/margins": 0.4935058653354645, + "rewards/rejected": -0.9156249761581421, + "step": 1620 + }, + { + "epoch": 0.41838311019567453, + "grad_norm": 516.0, + "learning_rate": 2.9080844490216274e-07, + "logits/chosen": -2.700000047683716, + "logits/rejected": -2.528125047683716, + "logps/chosen": -265.79998779296875, + "logps/rejected": -285.6000061035156, + "loss": 0.6043, + "rewards/accuracies": 0.6096795201301575, + "rewards/chosen": -0.531933605670929, + "rewards/margins": 0.678515613079071, + "rewards/rejected": -1.208593726158142, + "step": 1625 + }, + { + "epoch": 0.4196704428424305, + "grad_norm": 724.0, + "learning_rate": 2.9016477857878473e-07, + "logits/chosen": -2.7874999046325684, + "logits/rejected": -2.7562499046325684, + "logps/chosen": -334.3999938964844, + "logps/rejected": -329.20001220703125, + "loss": 0.6238, + "rewards/accuracies": 0.6581196784973145, + "rewards/chosen": -0.470458984375, + "rewards/margins": 0.601269543170929, + "rewards/rejected": -1.0725586414337158, + "step": 1630 + }, + { + "epoch": 0.4209577754891864, + "grad_norm": 472.0, + "learning_rate": 2.8952111225540677e-07, + "logits/chosen": -2.1734375953674316, + "logits/rejected": -2.167187452316284, + "logps/chosen": -291.20001220703125, + "logps/rejected": -375.75, + "loss": 0.4199, + "rewards/accuracies": 0.753333330154419, + "rewards/chosen": -0.6402343511581421, + "rewards/margins": 1.3777344226837158, + "rewards/rejected": -2.018749952316284, + "step": 1635 + }, + { + "epoch": 0.42224510813594235, + "grad_norm": 584.0, + "learning_rate": 2.8887744593202886e-07, + "logits/chosen": -2.918750047683716, + "logits/rejected": -2.7562499046325684, + "logps/chosen": -337.0, + "logps/rejected": -349.6000061035156, + "loss": 0.5043, + "rewards/accuracies": 0.7718182802200317, + "rewards/chosen": -0.10712890326976776, + "rewards/margins": 0.7066406011581421, + "rewards/rejected": -0.815625011920929, + "step": 1640 + }, + { + "epoch": 0.42353244078269825, + "grad_norm": 548.0, + "learning_rate": 2.8823377960865085e-07, + "logits/chosen": -2.71875, + "logits/rejected": -2.7406249046325684, + "logps/chosen": -322.3999938964844, + "logps/rejected": -343.6000061035156, + "loss": 0.6117, + "rewards/accuracies": 0.6493590474128723, + "rewards/chosen": -0.43486326932907104, + "rewards/margins": 0.5960937738418579, + "rewards/rejected": -1.0324218273162842, + "step": 1645 + }, + { + "epoch": 0.42481977342945415, + "grad_norm": 330.0, + "learning_rate": 2.875901132852729e-07, + "logits/chosen": -2.653125047683716, + "logits/rejected": -2.871875047683716, + "logps/chosen": -277.6000061035156, + "logps/rejected": -326.79998779296875, + "loss": 0.4734, + "rewards/accuracies": 0.7491666674613953, + "rewards/chosen": -0.3220458924770355, + "rewards/margins": 0.932421863079071, + "rewards/rejected": -1.255468726158142, + "step": 1650 + }, + { + "epoch": 0.4261071060762101, + "grad_norm": 588.0, + "learning_rate": 2.8694644696189494e-07, + "logits/chosen": -2.762500047683716, + "logits/rejected": -2.731250047683716, + "logps/chosen": -274.20001220703125, + "logps/rejected": -326.3999938964844, + "loss": 0.5148, + "rewards/accuracies": 0.7362271547317505, + "rewards/chosen": -0.41035157442092896, + "rewards/margins": 0.809374988079071, + "rewards/rejected": -1.217187523841858, + "step": 1655 + }, + { + "epoch": 0.427394438722966, + "grad_norm": 460.0, + "learning_rate": 2.86302780638517e-07, + "logits/chosen": -2.53125, + "logits/rejected": -2.6156249046325684, + "logps/chosen": -399.6000061035156, + "logps/rejected": -449.6000061035156, + "loss": 0.3996, + "rewards/accuracies": 0.8058333396911621, + "rewards/chosen": -0.62255859375, + "rewards/margins": 1.2468750476837158, + "rewards/rejected": -1.868749976158142, + "step": 1660 + }, + { + "epoch": 0.4286817713697219, + "grad_norm": 316.0, + "learning_rate": 2.85659114315139e-07, + "logits/chosen": -2.4375, + "logits/rejected": -2.432812452316284, + "logps/chosen": -288.6000061035156, + "logps/rejected": -415.79998779296875, + "loss": 0.4719, + "rewards/accuracies": 0.7099999785423279, + "rewards/chosen": -0.3622070252895355, + "rewards/margins": 0.946093738079071, + "rewards/rejected": -1.30859375, + "step": 1665 + }, + { + "epoch": 0.42996910401647787, + "grad_norm": 532.0, + "learning_rate": 2.8501544799176107e-07, + "logits/chosen": -2.668750047683716, + "logits/rejected": -2.59375, + "logps/chosen": -324.0, + "logps/rejected": -328.3999938964844, + "loss": 0.566, + "rewards/accuracies": 0.6933333277702332, + "rewards/chosen": -0.31025391817092896, + "rewards/margins": 0.6058593988418579, + "rewards/rejected": -0.91796875, + "step": 1670 + }, + { + "epoch": 0.43125643666323377, + "grad_norm": 1432.0, + "learning_rate": 2.843717816683831e-07, + "logits/chosen": -2.5625, + "logits/rejected": -2.653125047683716, + "logps/chosen": -244.1999969482422, + "logps/rejected": -231.1999969482422, + "loss": 0.6055, + "rewards/accuracies": 0.6245055198669434, + "rewards/chosen": -0.34288328886032104, + "rewards/margins": 0.577343761920929, + "rewards/rejected": -0.921679675579071, + "step": 1675 + }, + { + "epoch": 0.4325437693099897, + "grad_norm": 660.0, + "learning_rate": 2.837281153450051e-07, + "logits/chosen": -2.6812500953674316, + "logits/rejected": -2.6624999046325684, + "logps/chosen": -321.79998779296875, + "logps/rejected": -304.20001220703125, + "loss": 0.6172, + "rewards/accuracies": 0.6735714673995972, + "rewards/chosen": -0.8050781488418579, + "rewards/margins": 0.791796863079071, + "rewards/rejected": -1.597900390625, + "step": 1680 + }, + { + "epoch": 0.4338311019567456, + "grad_norm": 700.0, + "learning_rate": 2.830844490216272e-07, + "logits/chosen": -2.65625, + "logits/rejected": -2.7906250953674316, + "logps/chosen": -374.0, + "logps/rejected": -353.6000061035156, + "loss": 0.5797, + "rewards/accuracies": 0.7233333587646484, + "rewards/chosen": -0.5213378667831421, + "rewards/margins": 0.524218738079071, + "rewards/rejected": -1.044531226158142, + "step": 1685 + }, + { + "epoch": 0.4351184346035015, + "grad_norm": 480.0, + "learning_rate": 2.8244078269824924e-07, + "logits/chosen": -2.559375047683716, + "logits/rejected": -2.578125, + "logps/chosen": -347.6000061035156, + "logps/rejected": -394.0, + "loss": 0.5094, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.760937511920929, + "rewards/margins": 1.009374976158142, + "rewards/rejected": -1.7687499523162842, + "step": 1690 + }, + { + "epoch": 0.4364057672502575, + "grad_norm": 472.0, + "learning_rate": 2.8179711637487123e-07, + "logits/chosen": -2.512500047683716, + "logits/rejected": -2.4156250953674316, + "logps/chosen": -308.3999938964844, + "logps/rejected": -381.6000061035156, + "loss": 0.4492, + "rewards/accuracies": 0.76500004529953, + "rewards/chosen": -0.782031238079071, + "rewards/margins": 1.1964843273162842, + "rewards/rejected": -1.978906273841858, + "step": 1695 + }, + { + "epoch": 0.4376930998970134, + "grad_norm": 338.0, + "learning_rate": 2.8115345005149327e-07, + "logits/chosen": -2.5953125953674316, + "logits/rejected": -2.4625000953674316, + "logps/chosen": -312.0, + "logps/rejected": -315.20001220703125, + "loss": 0.5699, + "rewards/accuracies": 0.6664502024650574, + "rewards/chosen": -0.245849609375, + "rewards/margins": 0.5414062738418579, + "rewards/rejected": -0.786328136920929, + "step": 1700 + }, + { + "epoch": 0.4389804325437693, + "grad_norm": 360.0, + "learning_rate": 2.8050978372811537e-07, + "logits/chosen": -2.6968750953674316, + "logits/rejected": -2.484375, + "logps/chosen": -308.79998779296875, + "logps/rejected": -334.79998779296875, + "loss": 0.5555, + "rewards/accuracies": 0.6819780468940735, + "rewards/chosen": -0.40937501192092896, + "rewards/margins": 0.61328125, + "rewards/rejected": -1.023828148841858, + "step": 1705 + }, + { + "epoch": 0.44026776519052524, + "grad_norm": 744.0, + "learning_rate": 2.7986611740473736e-07, + "logits/chosen": -2.4625000953674316, + "logits/rejected": -2.4625000953674316, + "logps/chosen": -359.6000061035156, + "logps/rejected": -454.3999938964844, + "loss": 0.452, + "rewards/accuracies": 0.7283333539962769, + "rewards/chosen": -0.8412109613418579, + "rewards/margins": 1.3601562976837158, + "rewards/rejected": -2.19921875, + "step": 1710 + }, + { + "epoch": 0.44155509783728114, + "grad_norm": 556.0, + "learning_rate": 2.792224510813594e-07, + "logits/chosen": -2.768749952316284, + "logits/rejected": -2.0, + "logps/chosen": -267.79998779296875, + "logps/rejected": -377.6000061035156, + "loss": 0.4375, + "rewards/accuracies": 0.7547619938850403, + "rewards/chosen": -0.676562488079071, + "rewards/margins": 1.303125023841858, + "rewards/rejected": -1.978124976158142, + "step": 1715 + }, + { + "epoch": 0.4428424304840371, + "grad_norm": 472.0, + "learning_rate": 2.7857878475798144e-07, + "logits/chosen": -2.737499952316284, + "logits/rejected": -2.7281250953674316, + "logps/chosen": -378.79998779296875, + "logps/rejected": -377.20001220703125, + "loss": 0.5336, + "rewards/accuracies": 0.6256410479545593, + "rewards/chosen": -0.59375, + "rewards/margins": 0.752734363079071, + "rewards/rejected": -1.345971703529358, + "step": 1720 + }, + { + "epoch": 0.444129763130793, + "grad_norm": 432.0, + "learning_rate": 2.779351184346035e-07, + "logits/chosen": -2.6468749046325684, + "logits/rejected": -2.674999952316284, + "logps/chosen": -289.0, + "logps/rejected": -286.6000061035156, + "loss": 0.473, + "rewards/accuracies": 0.7344655990600586, + "rewards/chosen": -0.3490966856479645, + "rewards/margins": 0.8207031488418579, + "rewards/rejected": -1.1710937023162842, + "step": 1725 + }, + { + "epoch": 0.4454170957775489, + "grad_norm": 362.0, + "learning_rate": 2.7729145211122553e-07, + "logits/chosen": -2.7562499046325684, + "logits/rejected": -2.731250047683716, + "logps/chosen": -363.20001220703125, + "logps/rejected": -388.79998779296875, + "loss": 0.5695, + "rewards/accuracies": 0.7221428751945496, + "rewards/chosen": -0.5400390625, + "rewards/margins": 0.7925781011581421, + "rewards/rejected": -1.330468773841858, + "step": 1730 + }, + { + "epoch": 0.44670442842430486, + "grad_norm": 524.0, + "learning_rate": 2.7664778578784757e-07, + "logits/chosen": -2.543750047683716, + "logits/rejected": -2.292187452316284, + "logps/chosen": -266.20001220703125, + "logps/rejected": -254.39999389648438, + "loss": 0.5266, + "rewards/accuracies": 0.6657342910766602, + "rewards/chosen": -0.087890625, + "rewards/margins": 0.5921875238418579, + "rewards/rejected": -0.6800781488418579, + "step": 1735 + }, + { + "epoch": 0.44799176107106076, + "grad_norm": 596.0, + "learning_rate": 2.760041194644696e-07, + "logits/chosen": -2.549999952316284, + "logits/rejected": -2.4625000953674316, + "logps/chosen": -281.3999938964844, + "logps/rejected": -319.6000061035156, + "loss": 0.459, + "rewards/accuracies": 0.7589285969734192, + "rewards/chosen": -0.6285156011581421, + "rewards/margins": 1.193750023841858, + "rewards/rejected": -1.8234374523162842, + "step": 1740 + }, + { + "epoch": 0.44927909371781666, + "grad_norm": 470.0, + "learning_rate": 2.7536045314109166e-07, + "logits/chosen": -2.6812500953674316, + "logits/rejected": -2.828125, + "logps/chosen": -241.8000030517578, + "logps/rejected": -229.5, + "loss": 0.5832, + "rewards/accuracies": 0.5852273106575012, + "rewards/chosen": -0.35546875, + "rewards/margins": 0.53759765625, + "rewards/rejected": -0.892578125, + "step": 1745 + }, + { + "epoch": 0.4505664263645726, + "grad_norm": 576.0, + "learning_rate": 2.747167868177137e-07, + "logits/chosen": -2.6937499046325684, + "logits/rejected": -2.684375047683716, + "logps/chosen": -349.20001220703125, + "logps/rejected": -398.79998779296875, + "loss": 0.584, + "rewards/accuracies": 0.6666666865348816, + "rewards/chosen": -0.910937488079071, + "rewards/margins": 0.804492175579071, + "rewards/rejected": -1.713964819908142, + "step": 1750 + }, + { + "epoch": 0.4518537590113285, + "grad_norm": 464.0, + "learning_rate": 2.7407312049433574e-07, + "logits/chosen": -2.5687499046325684, + "logits/rejected": -2.8031249046325684, + "logps/chosen": -354.20001220703125, + "logps/rejected": -310.6000061035156, + "loss": 0.4828, + "rewards/accuracies": 0.7144697308540344, + "rewards/chosen": -0.22890624403953552, + "rewards/margins": 0.83984375, + "rewards/rejected": -1.068750023841858, + "step": 1755 + }, + { + "epoch": 0.45314109165808447, + "grad_norm": 868.0, + "learning_rate": 2.7342945417095773e-07, + "logits/chosen": -2.4937500953674316, + "logits/rejected": -2.4281249046325684, + "logps/chosen": -275.6000061035156, + "logps/rejected": -308.6000061035156, + "loss": 0.5582, + "rewards/accuracies": 0.6747403144836426, + "rewards/chosen": -0.70849609375, + "rewards/margins": 0.855273425579071, + "rewards/rejected": -1.5632812976837158, + "step": 1760 + }, + { + "epoch": 0.4544284243048404, + "grad_norm": 462.0, + "learning_rate": 2.727857878475798e-07, + "logits/chosen": -2.6812500953674316, + "logits/rejected": -2.6781249046325684, + "logps/chosen": -332.0, + "logps/rejected": -418.0, + "loss": 0.5359, + "rewards/accuracies": 0.6325000524520874, + "rewards/chosen": -0.55859375, + "rewards/margins": 0.7894531488418579, + "rewards/rejected": -1.348046898841858, + "step": 1765 + }, + { + "epoch": 0.4557157569515963, + "grad_norm": 504.0, + "learning_rate": 2.7214212152420187e-07, + "logits/chosen": -2.375, + "logits/rejected": -2.4281249046325684, + "logps/chosen": -313.20001220703125, + "logps/rejected": -358.0, + "loss": 0.4754, + "rewards/accuracies": 0.7216483354568481, + "rewards/chosen": -0.40410155057907104, + "rewards/margins": 0.9761718511581421, + "rewards/rejected": -1.380859375, + "step": 1770 + }, + { + "epoch": 0.45700308959835223, + "grad_norm": 456.0, + "learning_rate": 2.7149845520082386e-07, + "logits/chosen": -2.5, + "logits/rejected": -2.6875, + "logps/chosen": -365.20001220703125, + "logps/rejected": -342.3999938964844, + "loss": 0.4301, + "rewards/accuracies": 0.8111904859542847, + "rewards/chosen": -0.34257811307907104, + "rewards/margins": 1.142187476158142, + "rewards/rejected": -1.484765648841858, + "step": 1775 + }, + { + "epoch": 0.45829042224510813, + "grad_norm": 648.0, + "learning_rate": 2.708547888774459e-07, + "logits/chosen": -2.684375047683716, + "logits/rejected": -2.7093749046325684, + "logps/chosen": -337.20001220703125, + "logps/rejected": -371.6000061035156, + "loss": 0.5625, + "rewards/accuracies": 0.6292949318885803, + "rewards/chosen": -0.64306640625, + "rewards/margins": 0.7171875238418579, + "rewards/rejected": -1.359375, + "step": 1780 + }, + { + "epoch": 0.45957775489186403, + "grad_norm": 462.0, + "learning_rate": 2.70211122554068e-07, + "logits/chosen": -2.7281250953674316, + "logits/rejected": -2.6875, + "logps/chosen": -315.0, + "logps/rejected": -311.6000061035156, + "loss": 0.5125, + "rewards/accuracies": 0.7590476274490356, + "rewards/chosen": -0.4968505799770355, + "rewards/margins": 0.8949218988418579, + "rewards/rejected": -1.3914062976837158, + "step": 1785 + }, + { + "epoch": 0.46086508753862, + "grad_norm": 356.0, + "learning_rate": 2.6956745623069e-07, + "logits/chosen": -2.534374952316284, + "logits/rejected": -2.715625047683716, + "logps/chosen": -278.6000061035156, + "logps/rejected": -295.0, + "loss": 0.5477, + "rewards/accuracies": 0.745476245880127, + "rewards/chosen": -0.4947265684604645, + "rewards/margins": 1.138769507408142, + "rewards/rejected": -1.634765625, + "step": 1790 + }, + { + "epoch": 0.4621524201853759, + "grad_norm": 524.0, + "learning_rate": 2.6892378990731203e-07, + "logits/chosen": -2.653125047683716, + "logits/rejected": -2.674999952316284, + "logps/chosen": -314.20001220703125, + "logps/rejected": -343.20001220703125, + "loss": 0.5109, + "rewards/accuracies": 0.7075000405311584, + "rewards/chosen": -0.33281248807907104, + "rewards/margins": 0.9609375, + "rewards/rejected": -1.290429711341858, + "step": 1795 + }, + { + "epoch": 0.46343975283213185, + "grad_norm": 680.0, + "learning_rate": 2.6828012358393407e-07, + "logits/chosen": -2.6968750953674316, + "logits/rejected": -2.65625, + "logps/chosen": -280.0, + "logps/rejected": -370.0, + "loss": 0.5059, + "rewards/accuracies": 0.7315477132797241, + "rewards/chosen": -0.36381834745407104, + "rewards/margins": 1.05078125, + "rewards/rejected": -1.4140625, + "step": 1800 + }, + { + "epoch": 0.46472708547888775, + "grad_norm": 1080.0, + "learning_rate": 2.676364572605561e-07, + "logits/chosen": -2.4937500953674316, + "logits/rejected": -2.6656250953674316, + "logps/chosen": -211.8000030517578, + "logps/rejected": -238.89999389648438, + "loss": 0.5922, + "rewards/accuracies": 0.6584523916244507, + "rewards/chosen": -0.4072418212890625, + "rewards/margins": 0.599609375, + "rewards/rejected": -1.008203148841858, + "step": 1805 + }, + { + "epoch": 0.46601441812564365, + "grad_norm": 732.0, + "learning_rate": 2.6699279093717816e-07, + "logits/chosen": -2.20703125, + "logits/rejected": -2.073046922683716, + "logps/chosen": -253.8000030517578, + "logps/rejected": -317.20001220703125, + "loss": 0.4578, + "rewards/accuracies": 0.7440043687820435, + "rewards/chosen": -0.572265625, + "rewards/margins": 0.9871460199356079, + "rewards/rejected": -1.5578124523162842, + "step": 1810 + }, + { + "epoch": 0.4673017507723996, + "grad_norm": 406.0, + "learning_rate": 2.663491246138002e-07, + "logits/chosen": -2.59375, + "logits/rejected": -2.606250047683716, + "logps/chosen": -274.20001220703125, + "logps/rejected": -346.3999938964844, + "loss": 0.4844, + "rewards/accuracies": 0.7668589949607849, + "rewards/chosen": -0.546875, + "rewards/margins": 0.9468749761581421, + "rewards/rejected": -1.4914062023162842, + "step": 1815 + }, + { + "epoch": 0.4685890834191555, + "grad_norm": 334.0, + "learning_rate": 2.6570545829042224e-07, + "logits/chosen": -2.5718750953674316, + "logits/rejected": -2.4781250953674316, + "logps/chosen": -270.79998779296875, + "logps/rejected": -331.6000061035156, + "loss": 0.4941, + "rewards/accuracies": 0.7697436213493347, + "rewards/chosen": -0.31689453125, + "rewards/margins": 0.850781261920929, + "rewards/rejected": -1.16796875, + "step": 1820 + }, + { + "epoch": 0.4698764160659114, + "grad_norm": 652.0, + "learning_rate": 2.6506179196704423e-07, + "logits/chosen": -2.5484375953674316, + "logits/rejected": -2.5531249046325684, + "logps/chosen": -368.3999938964844, + "logps/rejected": -389.79998779296875, + "loss": 0.5266, + "rewards/accuracies": 0.7631060481071472, + "rewards/chosen": -0.6234375238418579, + "rewards/margins": 1.089453101158142, + "rewards/rejected": -1.7117187976837158, + "step": 1825 + }, + { + "epoch": 0.47116374871266736, + "grad_norm": 312.0, + "learning_rate": 2.6441812564366633e-07, + "logits/chosen": -2.549999952316284, + "logits/rejected": -2.659374952316284, + "logps/chosen": -244.39999389648438, + "logps/rejected": -312.0, + "loss": 0.4941, + "rewards/accuracies": 0.727857232093811, + "rewards/chosen": -0.566601574420929, + "rewards/margins": 1.1124999523162842, + "rewards/rejected": -1.6796875, + "step": 1830 + }, + { + "epoch": 0.47245108135942326, + "grad_norm": 468.0, + "learning_rate": 2.6377445932028837e-07, + "logits/chosen": -2.496875047683716, + "logits/rejected": -2.40625, + "logps/chosen": -309.6000061035156, + "logps/rejected": -350.0, + "loss": 0.4541, + "rewards/accuracies": 0.7350000143051147, + "rewards/chosen": -0.3681640625, + "rewards/margins": 1.049218773841858, + "rewards/rejected": -1.41796875, + "step": 1835 + }, + { + "epoch": 0.4737384140061792, + "grad_norm": 480.0, + "learning_rate": 2.6313079299691036e-07, + "logits/chosen": -2.6312499046325684, + "logits/rejected": -2.6500000953674316, + "logps/chosen": -327.20001220703125, + "logps/rejected": -293.6000061035156, + "loss": 0.4785, + "rewards/accuracies": 0.7542582750320435, + "rewards/chosen": -0.12080077826976776, + "rewards/margins": 0.925000011920929, + "rewards/rejected": -1.046875, + "step": 1840 + }, + { + "epoch": 0.4750257466529351, + "grad_norm": 300.0, + "learning_rate": 2.6248712667353246e-07, + "logits/chosen": -2.65625, + "logits/rejected": -2.7125000953674316, + "logps/chosen": -336.0, + "logps/rejected": -383.20001220703125, + "loss": 0.3447, + "rewards/accuracies": 0.8566666841506958, + "rewards/chosen": -0.4315429627895355, + "rewards/margins": 1.4148437976837158, + "rewards/rejected": -1.8488280773162842, + "step": 1845 + }, + { + "epoch": 0.476313079299691, + "grad_norm": 360.0, + "learning_rate": 2.618434603501545e-07, + "logits/chosen": -2.700000047683716, + "logits/rejected": -2.746875047683716, + "logps/chosen": -329.6000061035156, + "logps/rejected": -372.79998779296875, + "loss": 0.4645, + "rewards/accuracies": 0.720695972442627, + "rewards/chosen": -0.09982910007238388, + "rewards/margins": 1.075781226158142, + "rewards/rejected": -1.1789062023162842, + "step": 1850 + }, + { + "epoch": 0.477600411946447, + "grad_norm": 504.0, + "learning_rate": 2.611997940267765e-07, + "logits/chosen": -2.6187500953674316, + "logits/rejected": -2.731250047683716, + "logps/chosen": -249.5, + "logps/rejected": -273.0, + "loss": 0.5348, + "rewards/accuracies": 0.7009615302085876, + "rewards/chosen": -0.3426757752895355, + "rewards/margins": 0.8316406011581421, + "rewards/rejected": -1.1746094226837158, + "step": 1855 + }, + { + "epoch": 0.4788877445932029, + "grad_norm": 884.0, + "learning_rate": 2.6055612770339853e-07, + "logits/chosen": -2.5562500953674316, + "logits/rejected": -2.5875000953674316, + "logps/chosen": -203.3000030517578, + "logps/rejected": -252.60000610351562, + "loss": 0.5488, + "rewards/accuracies": 0.6704761981964111, + "rewards/chosen": -0.2928405702114105, + "rewards/margins": 0.671142578125, + "rewards/rejected": -0.962890625, + "step": 1860 + }, + { + "epoch": 0.4801750772399588, + "grad_norm": 532.0, + "learning_rate": 2.599124613800206e-07, + "logits/chosen": -2.590625047683716, + "logits/rejected": -2.625, + "logps/chosen": -332.6000061035156, + "logps/rejected": -379.20001220703125, + "loss": 0.5641, + "rewards/accuracies": 0.6758333444595337, + "rewards/chosen": -0.9789062738418579, + "rewards/margins": 0.92578125, + "rewards/rejected": -1.90234375, + "step": 1865 + }, + { + "epoch": 0.48146240988671474, + "grad_norm": 310.0, + "learning_rate": 2.592687950566426e-07, + "logits/chosen": -2.609375, + "logits/rejected": -2.621875047683716, + "logps/chosen": -338.0, + "logps/rejected": -412.79998779296875, + "loss": 0.373, + "rewards/accuracies": 0.7983333468437195, + "rewards/chosen": -0.733105480670929, + "rewards/margins": 1.32421875, + "rewards/rejected": -2.0562500953674316, + "step": 1870 + }, + { + "epoch": 0.48274974253347064, + "grad_norm": 510.0, + "learning_rate": 2.5862512873326466e-07, + "logits/chosen": -2.596874952316284, + "logits/rejected": -2.765625, + "logps/chosen": -387.79998779296875, + "logps/rejected": -349.8999938964844, + "loss": 0.5121, + "rewards/accuracies": 0.6909340620040894, + "rewards/chosen": -0.3426757752895355, + "rewards/margins": 0.723437488079071, + "rewards/rejected": -1.0655517578125, + "step": 1875 + }, + { + "epoch": 0.4840370751802266, + "grad_norm": 1072.0, + "learning_rate": 2.579814624098867e-07, + "logits/chosen": -2.5687499046325684, + "logits/rejected": -2.453125, + "logps/chosen": -192.4499969482422, + "logps/rejected": -218.39999389648438, + "loss": 0.5191, + "rewards/accuracies": 0.7549999952316284, + "rewards/chosen": -0.4105468690395355, + "rewards/margins": 0.873046875, + "rewards/rejected": -1.285546898841858, + "step": 1880 + }, + { + "epoch": 0.4853244078269825, + "grad_norm": 306.0, + "learning_rate": 2.5733779608650874e-07, + "logits/chosen": -2.543750047683716, + "logits/rejected": -2.512500047683716, + "logps/chosen": -346.3999938964844, + "logps/rejected": -333.20001220703125, + "loss": 0.55, + "rewards/accuracies": 0.7751623392105103, + "rewards/chosen": -0.2660156190395355, + "rewards/margins": 0.8115234375, + "rewards/rejected": -1.0773437023162842, + "step": 1885 + }, + { + "epoch": 0.4866117404737384, + "grad_norm": 680.0, + "learning_rate": 2.566941297631308e-07, + "logits/chosen": -2.421875, + "logits/rejected": -2.4593749046325684, + "logps/chosen": -304.6000061035156, + "logps/rejected": -382.0, + "loss": 0.4504, + "rewards/accuracies": 0.7735714912414551, + "rewards/chosen": -0.9546874761581421, + "rewards/margins": 1.333984375, + "rewards/rejected": -2.2890625, + "step": 1890 + }, + { + "epoch": 0.48789907312049435, + "grad_norm": 704.0, + "learning_rate": 2.5605046343975283e-07, + "logits/chosen": -2.1703124046325684, + "logits/rejected": -2.778125047683716, + "logps/chosen": -190.1999969482422, + "logps/rejected": -228.0, + "loss": 0.5617, + "rewards/accuracies": 0.6607142686843872, + "rewards/chosen": -0.328125, + "rewards/margins": 0.6097656488418579, + "rewards/rejected": -0.9390869140625, + "step": 1895 + }, + { + "epoch": 0.48918640576725025, + "grad_norm": 636.0, + "learning_rate": 2.5540679711637487e-07, + "logits/chosen": -2.7093749046325684, + "logits/rejected": -2.706249952316284, + "logps/chosen": -317.3999938964844, + "logps/rejected": -318.70001220703125, + "loss": 0.5656, + "rewards/accuracies": 0.621666669845581, + "rewards/chosen": -0.8353515863418579, + "rewards/margins": 0.632617175579071, + "rewards/rejected": -1.4672362804412842, + "step": 1900 + }, + { + "epoch": 0.49047373841400616, + "grad_norm": 556.0, + "learning_rate": 2.5476313079299686e-07, + "logits/chosen": -2.565624952316284, + "logits/rejected": -2.518749952316284, + "logps/chosen": -303.3999938964844, + "logps/rejected": -319.6000061035156, + "loss": 0.5777, + "rewards/accuracies": 0.646374523639679, + "rewards/chosen": -0.39277344942092896, + "rewards/margins": 0.6971191167831421, + "rewards/rejected": -1.088476538658142, + "step": 1905 + }, + { + "epoch": 0.4917610710607621, + "grad_norm": 536.0, + "learning_rate": 2.5411946446961896e-07, + "logits/chosen": -2.5406250953674316, + "logits/rejected": -2.434375047683716, + "logps/chosen": -329.20001220703125, + "logps/rejected": -405.20001220703125, + "loss": 0.4395, + "rewards/accuracies": 0.7644230723381042, + "rewards/chosen": -0.80615234375, + "rewards/margins": 1.208593726158142, + "rewards/rejected": -2.012500047683716, + "step": 1910 + }, + { + "epoch": 0.493048403707518, + "grad_norm": 498.0, + "learning_rate": 2.53475798146241e-07, + "logits/chosen": -2.4593749046325684, + "logits/rejected": -2.5875000953674316, + "logps/chosen": -291.20001220703125, + "logps/rejected": -386.0, + "loss": 0.4359, + "rewards/accuracies": 0.7459615468978882, + "rewards/chosen": -0.535449206829071, + "rewards/margins": 1.2820312976837158, + "rewards/rejected": -1.8171875476837158, + "step": 1915 + }, + { + "epoch": 0.49433573635427397, + "grad_norm": 316.0, + "learning_rate": 2.52832131822863e-07, + "logits/chosen": -2.559375047683716, + "logits/rejected": -2.575000047683716, + "logps/chosen": -330.3999938964844, + "logps/rejected": -356.79998779296875, + "loss": 0.4398, + "rewards/accuracies": 0.7994047999382019, + "rewards/chosen": -0.5528808832168579, + "rewards/margins": 1.0773437023162842, + "rewards/rejected": -1.630468726158142, + "step": 1920 + }, + { + "epoch": 0.49562306900102987, + "grad_norm": 824.0, + "learning_rate": 2.521884654994851e-07, + "logits/chosen": -2.6656250953674316, + "logits/rejected": -2.71875, + "logps/chosen": -302.0, + "logps/rejected": -319.6000061035156, + "loss": 0.5117, + "rewards/accuracies": 0.6835606694221497, + "rewards/chosen": -0.795117199420929, + "rewards/margins": 0.973437488079071, + "rewards/rejected": -1.771093726158142, + "step": 1925 + }, + { + "epoch": 0.49691040164778577, + "grad_norm": 620.0, + "learning_rate": 2.5154479917610713e-07, + "logits/chosen": -2.440624952316284, + "logits/rejected": -2.653125047683716, + "logps/chosen": -323.3999938964844, + "logps/rejected": -323.20001220703125, + "loss": 0.6145, + "rewards/accuracies": 0.6756044626235962, + "rewards/chosen": -0.9183593988418579, + "rewards/margins": 0.911328136920929, + "rewards/rejected": -1.829687476158142, + "step": 1930 + }, + { + "epoch": 0.4981977342945417, + "grad_norm": 1592.0, + "learning_rate": 2.509011328527291e-07, + "logits/chosen": NaN, + "logits/rejected": -2.659374952316284, + "logps/chosen": -246.8000030517578, + "logps/rejected": -235.0, + "loss": 0.6188, + "rewards/accuracies": 0.6389285922050476, + "rewards/chosen": -0.4164062440395355, + "rewards/margins": 0.672607421875, + "rewards/rejected": -1.0896484851837158, + "step": 1935 + }, + { + "epoch": 0.49948506694129763, + "grad_norm": 736.0, + "learning_rate": 2.5025746652935116e-07, + "logits/chosen": -2.5374999046325684, + "logits/rejected": -2.65625, + "logps/chosen": -280.3999938964844, + "logps/rejected": -322.20001220703125, + "loss": 0.5148, + "rewards/accuracies": 0.7044444680213928, + "rewards/chosen": -0.41669923067092896, + "rewards/margins": 0.847851574420929, + "rewards/rejected": -1.267187476158142, + "step": 1940 + }, + { + "epoch": 0.5007723995880535, + "grad_norm": 290.0, + "learning_rate": 2.496138002059732e-07, + "logits/chosen": -2.606250047683716, + "logits/rejected": -2.503124952316284, + "logps/chosen": -263.20001220703125, + "logps/rejected": -267.20001220703125, + "loss": 0.557, + "rewards/accuracies": 0.7234523892402649, + "rewards/chosen": -0.503710925579071, + "rewards/margins": 0.698046863079071, + "rewards/rejected": -1.1984374523162842, + "step": 1945 + }, + { + "epoch": 0.5020597322348095, + "grad_norm": 486.0, + "learning_rate": 2.4897013388259525e-07, + "logits/chosen": -2.5687499046325684, + "logits/rejected": -2.643749952316284, + "logps/chosen": -385.6000061035156, + "logps/rejected": -398.3999938964844, + "loss": 0.5137, + "rewards/accuracies": 0.7245238423347473, + "rewards/chosen": -0.6273437738418579, + "rewards/margins": 0.8500000238418579, + "rewards/rejected": -1.4765625, + "step": 1950 + }, + { + "epoch": 0.5033470648815654, + "grad_norm": 462.0, + "learning_rate": 2.483264675592173e-07, + "logits/chosen": -2.4000000953674316, + "logits/rejected": -2.424999952316284, + "logps/chosen": -290.79998779296875, + "logps/rejected": -328.79998779296875, + "loss": 0.5805, + "rewards/accuracies": 0.6553322076797485, + "rewards/chosen": -0.846484363079071, + "rewards/margins": 0.794921875, + "rewards/rejected": -1.642187476158142, + "step": 1955 + }, + { + "epoch": 0.5046343975283213, + "grad_norm": 548.0, + "learning_rate": 2.4768280123583933e-07, + "logits/chosen": -2.6156249046325684, + "logits/rejected": -2.559375047683716, + "logps/chosen": -326.79998779296875, + "logps/rejected": -450.79998779296875, + "loss": 0.4215, + "rewards/accuracies": 0.7230769395828247, + "rewards/chosen": -0.808667004108429, + "rewards/margins": 1.466406226158142, + "rewards/rejected": -2.272656202316284, + "step": 1960 + }, + { + "epoch": 0.5059217301750772, + "grad_norm": 704.0, + "learning_rate": 2.470391349124614e-07, + "logits/chosen": -2.6312499046325684, + "logits/rejected": -2.6624999046325684, + "logps/chosen": -271.79998779296875, + "logps/rejected": -273.20001220703125, + "loss": 0.7027, + "rewards/accuracies": 0.6361111402511597, + "rewards/chosen": -0.6031738519668579, + "rewards/margins": 0.3484741151332855, + "rewards/rejected": -0.9496093988418579, + "step": 1965 + }, + { + "epoch": 0.5072090628218332, + "grad_norm": 502.0, + "learning_rate": 2.463954685890834e-07, + "logits/chosen": -2.5562500953674316, + "logits/rejected": -2.643749952316284, + "logps/chosen": -364.0, + "logps/rejected": -349.6000061035156, + "loss": 0.5621, + "rewards/accuracies": 0.6610805988311768, + "rewards/chosen": -0.302734375, + "rewards/margins": 0.6240234375, + "rewards/rejected": -0.926953136920929, + "step": 1970 + }, + { + "epoch": 0.508496395468589, + "grad_norm": 536.0, + "learning_rate": 2.4575180226570546e-07, + "logits/chosen": -2.606250047683716, + "logits/rejected": -2.674999952316284, + "logps/chosen": -328.79998779296875, + "logps/rejected": -352.3999938964844, + "loss": 0.5879, + "rewards/accuracies": 0.6291667222976685, + "rewards/chosen": -0.527539074420929, + "rewards/margins": 0.6312500238418579, + "rewards/rejected": -1.157812476158142, + "step": 1975 + }, + { + "epoch": 0.509783728115345, + "grad_norm": 396.0, + "learning_rate": 2.451081359423275e-07, + "logits/chosen": -2.5875000953674316, + "logits/rejected": -2.590625047683716, + "logps/chosen": -319.79998779296875, + "logps/rejected": -423.6000061035156, + "loss": 0.5262, + "rewards/accuracies": 0.791282057762146, + "rewards/chosen": -0.7289062738418579, + "rewards/margins": 0.951171875, + "rewards/rejected": -1.6828124523162842, + "step": 1980 + }, + { + "epoch": 0.511071060762101, + "grad_norm": 608.0, + "learning_rate": 2.4446446961894955e-07, + "logits/chosen": -2.606250047683716, + "logits/rejected": -2.6875, + "logps/chosen": -296.79998779296875, + "logps/rejected": -377.3999938964844, + "loss": 0.5066, + "rewards/accuracies": 0.7452381253242493, + "rewards/chosen": -0.796215832233429, + "rewards/margins": 0.830859363079071, + "rewards/rejected": -1.625390648841858, + "step": 1985 + }, + { + "epoch": 0.5123583934088568, + "grad_norm": 516.0, + "learning_rate": 2.4382080329557153e-07, + "logits/chosen": -2.515625, + "logits/rejected": -2.317187547683716, + "logps/chosen": -340.79998779296875, + "logps/rejected": -333.20001220703125, + "loss": 0.4352, + "rewards/accuracies": 0.7746428847312927, + "rewards/chosen": -0.49702149629592896, + "rewards/margins": 1.0828125476837158, + "rewards/rejected": -1.5812499523162842, + "step": 1990 + }, + { + "epoch": 0.5136457260556128, + "grad_norm": 502.0, + "learning_rate": 2.4317713697219363e-07, + "logits/chosen": -2.7406249046325684, + "logits/rejected": -2.8499999046325684, + "logps/chosen": -283.79998779296875, + "logps/rejected": -349.79998779296875, + "loss": 0.643, + "rewards/accuracies": 0.633809506893158, + "rewards/chosen": -0.7022460699081421, + "rewards/margins": 0.43671876192092896, + "rewards/rejected": -1.1378905773162842, + "step": 1995 + }, + { + "epoch": 0.5149330587023687, + "grad_norm": 1096.0, + "learning_rate": 2.425334706488156e-07, + "logits/chosen": -2.778125047683716, + "logits/rejected": -2.481250047683716, + "logps/chosen": -282.79998779296875, + "logps/rejected": -281.79998779296875, + "loss": 0.5305, + "rewards/accuracies": 0.7749542593955994, + "rewards/chosen": -0.1767578125, + "rewards/margins": 0.832812488079071, + "rewards/rejected": -1.0087890625, + "step": 2000 + }, + { + "epoch": 0.5162203913491246, + "grad_norm": 688.0, + "learning_rate": 2.4188980432543766e-07, + "logits/chosen": -2.8343749046325684, + "logits/rejected": -2.796875, + "logps/chosen": -335.6000061035156, + "logps/rejected": -346.0, + "loss": 0.6148, + "rewards/accuracies": 0.6548810005187988, + "rewards/chosen": -0.6273437738418579, + "rewards/margins": 0.6167968511581421, + "rewards/rejected": -1.243749976158142, + "step": 2005 + }, + { + "epoch": 0.5175077239958805, + "grad_norm": 344.0, + "learning_rate": 2.412461380020597e-07, + "logits/chosen": -2.684375047683716, + "logits/rejected": -2.621875047683716, + "logps/chosen": -367.6000061035156, + "logps/rejected": -373.6000061035156, + "loss": 0.4922, + "rewards/accuracies": 0.7752380967140198, + "rewards/chosen": -0.6861327886581421, + "rewards/margins": 0.9087890386581421, + "rewards/rejected": -1.595312476158142, + "step": 2010 + }, + { + "epoch": 0.5187950566426365, + "grad_norm": 720.0, + "learning_rate": 2.4060247167868175e-07, + "logits/chosen": -2.6656250953674316, + "logits/rejected": -2.450000047683716, + "logps/chosen": -269.79998779296875, + "logps/rejected": -301.0, + "loss": 0.6895, + "rewards/accuracies": 0.6481060981750488, + "rewards/chosen": -0.64453125, + "rewards/margins": 0.5843750238418579, + "rewards/rejected": -1.228124976158142, + "step": 2015 + }, + { + "epoch": 0.5200823892893924, + "grad_norm": 808.0, + "learning_rate": 2.399588053553038e-07, + "logits/chosen": -2.6187500953674316, + "logits/rejected": -2.3671875, + "logps/chosen": -250.39999389648438, + "logps/rejected": -216.39999389648438, + "loss": 0.6953, + "rewards/accuracies": 0.5286111235618591, + "rewards/chosen": -0.13496093451976776, + "rewards/margins": 0.17626953125, + "rewards/rejected": -0.31098634004592896, + "step": 2020 + }, + { + "epoch": 0.5213697219361483, + "grad_norm": 628.0, + "learning_rate": 2.3931513903192583e-07, + "logits/chosen": -2.659374952316284, + "logits/rejected": -2.674999952316284, + "logps/chosen": -323.0, + "logps/rejected": -372.3999938964844, + "loss": 0.3652, + "rewards/accuracies": 0.8360713720321655, + "rewards/chosen": -0.6742187738418579, + "rewards/margins": 1.330468773841858, + "rewards/rejected": -2.0062499046325684, + "step": 2025 + }, + { + "epoch": 0.5226570545829042, + "grad_norm": 406.0, + "learning_rate": 2.386714727085479e-07, + "logits/chosen": -2.721874952316284, + "logits/rejected": -2.6656250953674316, + "logps/chosen": -359.20001220703125, + "logps/rejected": -391.79998779296875, + "loss": 0.4305, + "rewards/accuracies": 0.765097439289093, + "rewards/chosen": -0.5865234136581421, + "rewards/margins": 1.0890624523162842, + "rewards/rejected": -1.678125023841858, + "step": 2030 + }, + { + "epoch": 0.5239443872296602, + "grad_norm": 676.0, + "learning_rate": 2.3802780638516992e-07, + "logits/chosen": -2.609375, + "logits/rejected": -2.5531249046325684, + "logps/chosen": -262.5, + "logps/rejected": -260.6000061035156, + "loss": 0.5551, + "rewards/accuracies": 0.6433790922164917, + "rewards/chosen": -0.552050769329071, + "rewards/margins": 0.669726550579071, + "rewards/rejected": -1.222558617591858, + "step": 2035 + }, + { + "epoch": 0.525231719876416, + "grad_norm": 438.0, + "learning_rate": 2.3738414006179194e-07, + "logits/chosen": -2.456249952316284, + "logits/rejected": -2.301562547683716, + "logps/chosen": -296.79998779296875, + "logps/rejected": -342.6000061035156, + "loss": 0.577, + "rewards/accuracies": 0.633571445941925, + "rewards/chosen": -0.6929687261581421, + "rewards/margins": 0.803515613079071, + "rewards/rejected": -1.493749976158142, + "step": 2040 + }, + { + "epoch": 0.526519052523172, + "grad_norm": 478.0, + "learning_rate": 2.36740473738414e-07, + "logits/chosen": -2.546875, + "logits/rejected": -2.5843749046325684, + "logps/chosen": -362.0, + "logps/rejected": -373.20001220703125, + "loss": 0.4531, + "rewards/accuracies": 0.7350000143051147, + "rewards/chosen": -0.5211181640625, + "rewards/margins": 0.9976562261581421, + "rewards/rejected": -1.5203125476837158, + "step": 2045 + }, + { + "epoch": 0.527806385169928, + "grad_norm": 438.0, + "learning_rate": 2.3609680741503605e-07, + "logits/chosen": -2.6875, + "logits/rejected": -2.325000047683716, + "logps/chosen": -277.20001220703125, + "logps/rejected": -281.20001220703125, + "loss": 0.4857, + "rewards/accuracies": 0.7050000429153442, + "rewards/chosen": -0.3549438416957855, + "rewards/margins": 0.8968750238418579, + "rewards/rejected": -1.251562476158142, + "step": 2050 + }, + { + "epoch": 0.5290937178166838, + "grad_norm": 516.0, + "learning_rate": 2.3545314109165806e-07, + "logits/chosen": -2.5921874046325684, + "logits/rejected": -2.504687547683716, + "logps/chosen": -299.20001220703125, + "logps/rejected": -314.6000061035156, + "loss": 0.5746, + "rewards/accuracies": 0.6457400918006897, + "rewards/chosen": -0.645703136920929, + "rewards/margins": 0.894335925579071, + "rewards/rejected": -1.5390625, + "step": 2055 + }, + { + "epoch": 0.5303810504634398, + "grad_norm": 516.0, + "learning_rate": 2.348094747682801e-07, + "logits/chosen": -2.6031250953674316, + "logits/rejected": -2.620312452316284, + "logps/chosen": -304.20001220703125, + "logps/rejected": -331.3999938964844, + "loss": 0.4135, + "rewards/accuracies": 0.8141666650772095, + "rewards/chosen": -0.533984363079071, + "rewards/margins": 1.1960937976837158, + "rewards/rejected": -1.728906273841858, + "step": 2060 + }, + { + "epoch": 0.5316683831101957, + "grad_norm": 446.0, + "learning_rate": 2.3416580844490218e-07, + "logits/chosen": -2.590625047683716, + "logits/rejected": -2.6624999046325684, + "logps/chosen": -332.79998779296875, + "logps/rejected": -370.79998779296875, + "loss": 0.5539, + "rewards/accuracies": 0.6803571581840515, + "rewards/chosen": -0.555859386920929, + "rewards/margins": 0.7841796875, + "rewards/rejected": -1.3390624523162842, + "step": 2065 + }, + { + "epoch": 0.5329557157569516, + "grad_norm": 596.0, + "learning_rate": 2.335221421215242e-07, + "logits/chosen": -2.637500047683716, + "logits/rejected": -2.706249952316284, + "logps/chosen": -290.79998779296875, + "logps/rejected": -351.79998779296875, + "loss": 0.4703, + "rewards/accuracies": 0.7355555295944214, + "rewards/chosen": -0.6875, + "rewards/margins": 0.955517590045929, + "rewards/rejected": -1.6414062976837158, + "step": 2070 + }, + { + "epoch": 0.5342430484037075, + "grad_norm": 768.0, + "learning_rate": 2.3287847579814623e-07, + "logits/chosen": -2.8968749046325684, + "logits/rejected": -2.6859374046325684, + "logps/chosen": -240.10000610351562, + "logps/rejected": -227.1999969482422, + "loss": 0.6367, + "rewards/accuracies": 0.597644031047821, + "rewards/chosen": -0.3270019590854645, + "rewards/margins": 0.3857421875, + "rewards/rejected": -0.712646484375, + "step": 2075 + }, + { + "epoch": 0.5355303810504635, + "grad_norm": 434.0, + "learning_rate": 2.3223480947476825e-07, + "logits/chosen": -2.596874952316284, + "logits/rejected": -2.465625047683716, + "logps/chosen": -370.79998779296875, + "logps/rejected": -327.0, + "loss": 0.543, + "rewards/accuracies": 0.7000001072883606, + "rewards/chosen": -0.5126953125, + "rewards/margins": 0.6244140863418579, + "rewards/rejected": -1.135156273841858, + "step": 2080 + }, + { + "epoch": 0.5368177136972193, + "grad_norm": 708.0, + "learning_rate": 2.315911431513903e-07, + "logits/chosen": -2.6656250953674316, + "logits/rejected": -2.453125, + "logps/chosen": -337.6000061035156, + "logps/rejected": -331.6000061035156, + "loss": 0.6539, + "rewards/accuracies": 0.6375000476837158, + "rewards/chosen": -0.5521484613418579, + "rewards/margins": 0.4105468690395355, + "rewards/rejected": -0.961718738079071, + "step": 2085 + }, + { + "epoch": 0.5381050463439753, + "grad_norm": 892.0, + "learning_rate": 2.3094747682801236e-07, + "logits/chosen": -2.456249952316284, + "logits/rejected": -2.3765625953674316, + "logps/chosen": -265.79998779296875, + "logps/rejected": -252.0, + "loss": 0.5434, + "rewards/accuracies": 0.6456350088119507, + "rewards/chosen": -0.47539061307907104, + "rewards/margins": 0.6568359136581421, + "rewards/rejected": -1.1316406726837158, + "step": 2090 + }, + { + "epoch": 0.5393923789907312, + "grad_norm": 512.0, + "learning_rate": 2.3030381050463438e-07, + "logits/chosen": -2.684375047683716, + "logits/rejected": -2.609375, + "logps/chosen": -300.20001220703125, + "logps/rejected": -347.6000061035156, + "loss": 0.5551, + "rewards/accuracies": 0.721666693687439, + "rewards/chosen": -0.47587889432907104, + "rewards/margins": 0.7933593988418579, + "rewards/rejected": -1.26953125, + "step": 2095 + }, + { + "epoch": 0.5406797116374872, + "grad_norm": 536.0, + "learning_rate": 2.2966014418125642e-07, + "logits/chosen": -2.721874952316284, + "logits/rejected": -2.7437500953674316, + "logps/chosen": -307.0, + "logps/rejected": -322.3999938964844, + "loss": 0.5449, + "rewards/accuracies": 0.6821154356002808, + "rewards/chosen": -0.636425793170929, + "rewards/margins": 0.7250000238418579, + "rewards/rejected": -1.360937476158142, + "step": 2100 + }, + { + "epoch": 0.541967044284243, + "grad_norm": 916.0, + "learning_rate": 2.2901647785787846e-07, + "logits/chosen": -2.6500000953674316, + "logits/rejected": -2.6624999046325684, + "logps/chosen": -356.0, + "logps/rejected": -416.3999938964844, + "loss": 0.4313, + "rewards/accuracies": 0.8093253970146179, + "rewards/chosen": -0.780078113079071, + "rewards/margins": 1.17578125, + "rewards/rejected": -1.955468773841858, + "step": 2105 + }, + { + "epoch": 0.543254376930999, + "grad_norm": 572.0, + "learning_rate": 2.283728115345005e-07, + "logits/chosen": -2.7093749046325684, + "logits/rejected": -2.7437500953674316, + "logps/chosen": -315.20001220703125, + "logps/rejected": -288.20001220703125, + "loss": 0.5965, + "rewards/accuracies": 0.6264718770980835, + "rewards/chosen": -0.3970703184604645, + "rewards/margins": 0.683398425579071, + "rewards/rejected": -1.078515648841858, + "step": 2110 + }, + { + "epoch": 0.5445417095777549, + "grad_norm": 688.0, + "learning_rate": 2.2772914521112255e-07, + "logits/chosen": -2.6343750953674316, + "logits/rejected": -2.5687499046325684, + "logps/chosen": -331.20001220703125, + "logps/rejected": -370.3999938964844, + "loss": 0.4742, + "rewards/accuracies": 0.7752381563186646, + "rewards/chosen": -0.3993164002895355, + "rewards/margins": 1.067968726158142, + "rewards/rejected": -1.467187523841858, + "step": 2115 + }, + { + "epoch": 0.5458290422245108, + "grad_norm": 364.0, + "learning_rate": 2.2708547888774457e-07, + "logits/chosen": -2.734375, + "logits/rejected": -2.762500047683716, + "logps/chosen": -306.6000061035156, + "logps/rejected": -334.0, + "loss": 0.518, + "rewards/accuracies": 0.7241758108139038, + "rewards/chosen": -0.601318359375, + "rewards/margins": 0.962109386920929, + "rewards/rejected": -1.564062476158142, + "step": 2120 + }, + { + "epoch": 0.5471163748712667, + "grad_norm": 580.0, + "learning_rate": 2.264418125643666e-07, + "logits/chosen": -2.75, + "logits/rejected": -2.6968750953674316, + "logps/chosen": -349.79998779296875, + "logps/rejected": -385.6000061035156, + "loss": 0.4504, + "rewards/accuracies": 0.7933333516120911, + "rewards/chosen": -0.3070312440395355, + "rewards/margins": 0.9320312738418579, + "rewards/rejected": -1.2365233898162842, + "step": 2125 + }, + { + "epoch": 0.5484037075180227, + "grad_norm": 864.0, + "learning_rate": 2.2579814624098868e-07, + "logits/chosen": -2.4000000953674316, + "logits/rejected": -2.3499999046325684, + "logps/chosen": -335.6000061035156, + "logps/rejected": -305.20001220703125, + "loss": 0.5891, + "rewards/accuracies": 0.6761904954910278, + "rewards/chosen": -0.515429675579071, + "rewards/margins": 0.7230468988418579, + "rewards/rejected": -1.2390625476837158, + "step": 2130 + }, + { + "epoch": 0.5496910401647785, + "grad_norm": 334.0, + "learning_rate": 2.251544799176107e-07, + "logits/chosen": -2.6312499046325684, + "logits/rejected": -2.6312499046325684, + "logps/chosen": -351.20001220703125, + "logps/rejected": -376.79998779296875, + "loss": 0.5117, + "rewards/accuracies": 0.7158333659172058, + "rewards/chosen": -0.682373046875, + "rewards/margins": 0.98046875, + "rewards/rejected": -1.665624976158142, + "step": 2135 + }, + { + "epoch": 0.5509783728115345, + "grad_norm": 312.0, + "learning_rate": 2.2451081359423274e-07, + "logits/chosen": -2.512500047683716, + "logits/rejected": -2.6031250953674316, + "logps/chosen": -261.3999938964844, + "logps/rejected": -352.0, + "loss": 0.4193, + "rewards/accuracies": 0.7719481587409973, + "rewards/chosen": -0.561328113079071, + "rewards/margins": 1.191015601158142, + "rewards/rejected": -1.751562476158142, + "step": 2140 + }, + { + "epoch": 0.5522657054582905, + "grad_norm": 560.0, + "learning_rate": 2.2386714727085478e-07, + "logits/chosen": -2.6312499046325684, + "logits/rejected": -2.703125, + "logps/chosen": -350.6000061035156, + "logps/rejected": -318.3999938964844, + "loss": 0.5828, + "rewards/accuracies": 0.6303571462631226, + "rewards/chosen": -0.4156250059604645, + "rewards/margins": 0.500781238079071, + "rewards/rejected": -0.9156249761581421, + "step": 2145 + }, + { + "epoch": 0.5535530381050463, + "grad_norm": 680.0, + "learning_rate": 2.2322348094747682e-07, + "logits/chosen": -2.6656250953674316, + "logits/rejected": -2.734375, + "logps/chosen": -300.79998779296875, + "logps/rejected": -376.20001220703125, + "loss": 0.4695, + "rewards/accuracies": 0.7133333086967468, + "rewards/chosen": -0.47246092557907104, + "rewards/margins": 1.140625, + "rewards/rejected": -1.615625023841858, + "step": 2150 + }, + { + "epoch": 0.5548403707518023, + "grad_norm": 720.0, + "learning_rate": 2.2257981462409886e-07, + "logits/chosen": -2.612499952316284, + "logits/rejected": -2.5390625, + "logps/chosen": -354.79998779296875, + "logps/rejected": -383.3999938964844, + "loss": 0.5992, + "rewards/accuracies": 0.6486111283302307, + "rewards/chosen": -0.9986327886581421, + "rewards/margins": 0.777539074420929, + "rewards/rejected": -1.7742187976837158, + "step": 2155 + }, + { + "epoch": 0.5561277033985582, + "grad_norm": 748.0, + "learning_rate": 2.219361483007209e-07, + "logits/chosen": -2.4906249046325684, + "logits/rejected": -2.5, + "logps/chosen": -323.79998779296875, + "logps/rejected": -379.6000061035156, + "loss": 0.4891, + "rewards/accuracies": 0.71833336353302, + "rewards/chosen": -0.3660644590854645, + "rewards/margins": 0.9222656488418579, + "rewards/rejected": -1.2898437976837158, + "step": 2160 + }, + { + "epoch": 0.5574150360453141, + "grad_norm": 1012.0, + "learning_rate": 2.2129248197734292e-07, + "logits/chosen": -2.6156249046325684, + "logits/rejected": -2.53125, + "logps/chosen": -348.5, + "logps/rejected": -353.3999938964844, + "loss": 0.6406, + "rewards/accuracies": 0.6492856740951538, + "rewards/chosen": -0.7630859613418579, + "rewards/margins": 0.8472656011581421, + "rewards/rejected": -1.6101562976837158, + "step": 2165 + }, + { + "epoch": 0.55870236869207, + "grad_norm": 732.0, + "learning_rate": 2.20648815653965e-07, + "logits/chosen": -2.575000047683716, + "logits/rejected": -2.653125047683716, + "logps/chosen": -285.3999938964844, + "logps/rejected": -300.20001220703125, + "loss": 0.5156, + "rewards/accuracies": 0.7637063264846802, + "rewards/chosen": -0.4610595703125, + "rewards/margins": 0.7920898199081421, + "rewards/rejected": -1.25390625, + "step": 2170 + }, + { + "epoch": 0.559989701338826, + "grad_norm": 464.0, + "learning_rate": 2.20005149330587e-07, + "logits/chosen": -2.621875047683716, + "logits/rejected": -2.487499952316284, + "logps/chosen": -306.6000061035156, + "logps/rejected": -333.20001220703125, + "loss": 0.5027, + "rewards/accuracies": 0.7009615302085876, + "rewards/chosen": -0.745312511920929, + "rewards/margins": 1.12109375, + "rewards/rejected": -1.8671875, + "step": 2175 + }, + { + "epoch": 0.5612770339855818, + "grad_norm": 462.0, + "learning_rate": 2.1936148300720905e-07, + "logits/chosen": -2.590625047683716, + "logits/rejected": -2.628124952316284, + "logps/chosen": -293.20001220703125, + "logps/rejected": -364.0, + "loss": 0.448, + "rewards/accuracies": 0.7428571581840515, + "rewards/chosen": -0.561718761920929, + "rewards/margins": 1.20703125, + "rewards/rejected": -1.771093726158142, + "step": 2180 + }, + { + "epoch": 0.5625643666323378, + "grad_norm": 852.0, + "learning_rate": 2.187178166838311e-07, + "logits/chosen": -2.6343750953674316, + "logits/rejected": -2.653125047683716, + "logps/chosen": -265.79998779296875, + "logps/rejected": -355.6000061035156, + "loss": 0.4598, + "rewards/accuracies": 0.6428571939468384, + "rewards/chosen": -0.5296875238418579, + "rewards/margins": 1.2257812023162842, + "rewards/rejected": -1.7578125, + "step": 2185 + }, + { + "epoch": 0.5638516992790937, + "grad_norm": 896.0, + "learning_rate": 2.1807415036045314e-07, + "logits/chosen": -2.5609374046325684, + "logits/rejected": -2.479687452316284, + "logps/chosen": -322.79998779296875, + "logps/rejected": -378.3999938964844, + "loss": 0.4891, + "rewards/accuracies": 0.712692379951477, + "rewards/chosen": -0.5005859136581421, + "rewards/margins": 0.965624988079071, + "rewards/rejected": -1.4660155773162842, + "step": 2190 + }, + { + "epoch": 0.5651390319258497, + "grad_norm": 432.0, + "learning_rate": 2.1743048403707518e-07, + "logits/chosen": -2.6656250953674316, + "logits/rejected": NaN, + "logps/chosen": -285.6000061035156, + "logps/rejected": -311.6000061035156, + "loss": 0.5375, + "rewards/accuracies": 0.7444047927856445, + "rewards/chosen": -0.51123046875, + "rewards/margins": 0.75341796875, + "rewards/rejected": -1.264062523841858, + "step": 2195 + }, + { + "epoch": 0.5664263645726055, + "grad_norm": 556.0, + "learning_rate": 2.1678681771369722e-07, + "logits/chosen": -2.59375, + "logits/rejected": -2.512500047683716, + "logps/chosen": -321.20001220703125, + "logps/rejected": -375.6000061035156, + "loss": 0.5488, + "rewards/accuracies": 0.667362630367279, + "rewards/chosen": -0.8089843988418579, + "rewards/margins": 0.864453136920929, + "rewards/rejected": -1.673437476158142, + "step": 2200 + }, + { + "epoch": 0.5677136972193615, + "grad_norm": 366.0, + "learning_rate": 2.1614315139031924e-07, + "logits/chosen": -2.762500047683716, + "logits/rejected": -2.7593750953674316, + "logps/chosen": -313.20001220703125, + "logps/rejected": -347.20001220703125, + "loss": 0.6492, + "rewards/accuracies": 0.594468891620636, + "rewards/chosen": -0.2850585877895355, + "rewards/margins": 0.4730468690395355, + "rewards/rejected": -0.756640613079071, + "step": 2205 + }, + { + "epoch": 0.5690010298661174, + "grad_norm": 756.0, + "learning_rate": 2.154994850669413e-07, + "logits/chosen": -2.5234375, + "logits/rejected": -2.5093750953674316, + "logps/chosen": -316.3999938964844, + "logps/rejected": -338.6000061035156, + "loss": 0.5188, + "rewards/accuracies": 0.8067307472229004, + "rewards/chosen": -0.6103515625, + "rewards/margins": 0.9320312738418579, + "rewards/rejected": -1.544531226158142, + "step": 2210 + }, + { + "epoch": 0.5702883625128733, + "grad_norm": 504.0, + "learning_rate": 2.1485581874356332e-07, + "logits/chosen": -2.643749952316284, + "logits/rejected": -2.628124952316284, + "logps/chosen": -300.3999938964844, + "logps/rejected": -359.20001220703125, + "loss": 0.4688, + "rewards/accuracies": 0.7376190423965454, + "rewards/chosen": -0.3428710997104645, + "rewards/margins": 0.98828125, + "rewards/rejected": -1.3289062976837158, + "step": 2215 + }, + { + "epoch": 0.5715756951596292, + "grad_norm": 490.0, + "learning_rate": 2.1421215242018537e-07, + "logits/chosen": -2.3218750953674316, + "logits/rejected": -2.4359374046325684, + "logps/chosen": -277.0, + "logps/rejected": -344.3999938964844, + "loss": 0.5195, + "rewards/accuracies": 0.6911905407905579, + "rewards/chosen": -0.69677734375, + "rewards/margins": 0.991015613079071, + "rewards/rejected": -1.6867187023162842, + "step": 2220 + }, + { + "epoch": 0.5728630278063852, + "grad_norm": 556.0, + "learning_rate": 2.135684860968074e-07, + "logits/chosen": -2.721874952316284, + "logits/rejected": -2.546875, + "logps/chosen": -273.0, + "logps/rejected": -358.3999938964844, + "loss": 0.5289, + "rewards/accuracies": 0.6664502024650574, + "rewards/chosen": -0.4420410096645355, + "rewards/margins": 0.8160156011581421, + "rewards/rejected": -1.2566406726837158, + "step": 2225 + }, + { + "epoch": 0.574150360453141, + "grad_norm": 668.0, + "learning_rate": 2.1292481977342942e-07, + "logits/chosen": -2.5843749046325684, + "logits/rejected": -2.487499952316284, + "logps/chosen": -304.20001220703125, + "logps/rejected": -331.6000061035156, + "loss": 0.4703, + "rewards/accuracies": 0.816620945930481, + "rewards/chosen": -0.2972656190395355, + "rewards/margins": 0.93359375, + "rewards/rejected": -1.2296874523162842, + "step": 2230 + }, + { + "epoch": 0.575437693099897, + "grad_norm": 628.0, + "learning_rate": 2.122811534500515e-07, + "logits/chosen": -2.5640625953674316, + "logits/rejected": -2.621875047683716, + "logps/chosen": -328.20001220703125, + "logps/rejected": -315.3999938964844, + "loss": 0.5652, + "rewards/accuracies": 0.6893590092658997, + "rewards/chosen": -0.36761474609375, + "rewards/margins": 0.557421863079071, + "rewards/rejected": -0.926562488079071, + "step": 2235 + }, + { + "epoch": 0.576725025746653, + "grad_norm": 596.0, + "learning_rate": 2.1163748712667354e-07, + "logits/chosen": -2.856250047683716, + "logits/rejected": -2.6500000953674316, + "logps/chosen": -251.3000030517578, + "logps/rejected": -240.10000610351562, + "loss": 0.584, + "rewards/accuracies": 0.5659340620040894, + "rewards/chosen": -0.3447265625, + "rewards/margins": 0.4162841737270355, + "rewards/rejected": -0.7621093988418579, + "step": 2240 + }, + { + "epoch": 0.5780123583934088, + "grad_norm": 556.0, + "learning_rate": 2.1099382080329555e-07, + "logits/chosen": -2.6187500953674316, + "logits/rejected": -2.715625047683716, + "logps/chosen": -311.20001220703125, + "logps/rejected": -327.70001220703125, + "loss": 0.5563, + "rewards/accuracies": 0.6720887422561646, + "rewards/chosen": -0.498046875, + "rewards/margins": 0.9296875, + "rewards/rejected": -1.42919921875, + "step": 2245 + }, + { + "epoch": 0.5792996910401648, + "grad_norm": 372.0, + "learning_rate": 2.103501544799176e-07, + "logits/chosen": -2.4609375, + "logits/rejected": -2.7906250953674316, + "logps/chosen": -288.6000061035156, + "logps/rejected": -378.20001220703125, + "loss": 0.5809, + "rewards/accuracies": 0.723141074180603, + "rewards/chosen": -0.571826159954071, + "rewards/margins": 0.6031249761581421, + "rewards/rejected": -1.1749999523162842, + "step": 2250 + }, + { + "epoch": 0.5805870236869207, + "grad_norm": 400.0, + "learning_rate": 2.0970648815653964e-07, + "logits/chosen": -2.6624999046325684, + "logits/rejected": -2.606250047683716, + "logps/chosen": -382.20001220703125, + "logps/rejected": -351.79998779296875, + "loss": 0.5012, + "rewards/accuracies": 0.6797435879707336, + "rewards/chosen": -0.4488281309604645, + "rewards/margins": 0.811718761920929, + "rewards/rejected": -1.2625000476837158, + "step": 2255 + }, + { + "epoch": 0.5818743563336766, + "grad_norm": 564.0, + "learning_rate": 2.0906282183316168e-07, + "logits/chosen": -2.512500047683716, + "logits/rejected": -2.534374952316284, + "logps/chosen": -286.20001220703125, + "logps/rejected": -315.20001220703125, + "loss": 0.4586, + "rewards/accuracies": 0.7062879204750061, + "rewards/chosen": -0.37187498807907104, + "rewards/margins": 1.0703125, + "rewards/rejected": -1.443750023841858, + "step": 2260 + }, + { + "epoch": 0.5831616889804325, + "grad_norm": 1312.0, + "learning_rate": 2.0841915550978372e-07, + "logits/chosen": -2.418750047683716, + "logits/rejected": -2.715625047683716, + "logps/chosen": -283.6000061035156, + "logps/rejected": -275.29998779296875, + "loss": 0.6402, + "rewards/accuracies": 0.5674242377281189, + "rewards/chosen": -0.37294310331344604, + "rewards/margins": 0.4572997987270355, + "rewards/rejected": -0.8296874761581421, + "step": 2265 + }, + { + "epoch": 0.5844490216271885, + "grad_norm": 520.0, + "learning_rate": 2.0777548918640574e-07, + "logits/chosen": -2.421875, + "logits/rejected": -2.5062499046325684, + "logps/chosen": -294.79998779296875, + "logps/rejected": -356.20001220703125, + "loss": 0.4566, + "rewards/accuracies": 0.761071503162384, + "rewards/chosen": -0.7789062261581421, + "rewards/margins": 1.263281226158142, + "rewards/rejected": -2.042187452316284, + "step": 2270 + }, + { + "epoch": 0.5857363542739444, + "grad_norm": 414.0, + "learning_rate": 2.071318228630278e-07, + "logits/chosen": -2.6343750953674316, + "logits/rejected": -2.684375047683716, + "logps/chosen": -325.20001220703125, + "logps/rejected": -359.79998779296875, + "loss": 0.5227, + "rewards/accuracies": 0.6562637090682983, + "rewards/chosen": -0.2784179747104645, + "rewards/margins": 0.7601562738418579, + "rewards/rejected": -1.0398437976837158, + "step": 2275 + }, + { + "epoch": 0.5870236869207003, + "grad_norm": 708.0, + "learning_rate": 2.0648815653964985e-07, + "logits/chosen": -2.2203125953674316, + "logits/rejected": -2.331249952316284, + "logps/chosen": -302.6000061035156, + "logps/rejected": -364.0, + "loss": 0.4566, + "rewards/accuracies": 0.7511904835700989, + "rewards/chosen": -1.2482421398162842, + "rewards/margins": 1.2117187976837158, + "rewards/rejected": -2.4593749046325684, + "step": 2280 + }, + { + "epoch": 0.5883110195674562, + "grad_norm": 394.0, + "learning_rate": 2.0584449021627187e-07, + "logits/chosen": -2.6312499046325684, + "logits/rejected": -2.6187500953674316, + "logps/chosen": -321.3999938964844, + "logps/rejected": -319.6000061035156, + "loss": 0.5, + "rewards/accuracies": 0.7244480848312378, + "rewards/chosen": -0.31242674589157104, + "rewards/margins": 0.8101562261581421, + "rewards/rejected": -1.1222655773162842, + "step": 2285 + }, + { + "epoch": 0.5895983522142122, + "grad_norm": 616.0, + "learning_rate": 2.052008238928939e-07, + "logits/chosen": -2.700000047683716, + "logits/rejected": -2.715625047683716, + "logps/chosen": -314.0, + "logps/rejected": -367.0, + "loss": 0.5176, + "rewards/accuracies": 0.7089285850524902, + "rewards/chosen": -0.4878906309604645, + "rewards/margins": 0.778124988079071, + "rewards/rejected": -1.266992211341858, + "step": 2290 + }, + { + "epoch": 0.590885684860968, + "grad_norm": 456.0, + "learning_rate": 2.0455715756951595e-07, + "logits/chosen": -2.637500047683716, + "logits/rejected": -2.6875, + "logps/chosen": -282.79998779296875, + "logps/rejected": -336.0, + "loss": 0.5711, + "rewards/accuracies": 0.645714282989502, + "rewards/chosen": -0.47802734375, + "rewards/margins": 0.6807616949081421, + "rewards/rejected": -1.1570312976837158, + "step": 2295 + }, + { + "epoch": 0.592173017507724, + "grad_norm": 836.0, + "learning_rate": 2.03913491246138e-07, + "logits/chosen": -2.5093750953674316, + "logits/rejected": -2.628124952316284, + "logps/chosen": -310.0, + "logps/rejected": -330.3999938964844, + "loss": 0.5031, + "rewards/accuracies": 0.7160714268684387, + "rewards/chosen": -0.3282226622104645, + "rewards/margins": 0.7300781011581421, + "rewards/rejected": -1.0578124523162842, + "step": 2300 + }, + { + "epoch": 0.59346035015448, + "grad_norm": 416.0, + "learning_rate": 2.0326982492276004e-07, + "logits/chosen": -2.7281250953674316, + "logits/rejected": -2.621875047683716, + "logps/chosen": -350.0, + "logps/rejected": -393.0, + "loss": 0.4891, + "rewards/accuracies": 0.7413889169692993, + "rewards/chosen": -0.4759765565395355, + "rewards/margins": 0.8280273675918579, + "rewards/rejected": -1.306640625, + "step": 2305 + }, + { + "epoch": 0.5947476828012358, + "grad_norm": 516.0, + "learning_rate": 2.0262615859938205e-07, + "logits/chosen": -2.5718750953674316, + "logits/rejected": -2.6312499046325684, + "logps/chosen": -388.3999938964844, + "logps/rejected": -462.79998779296875, + "loss": 0.4107, + "rewards/accuracies": 0.84416663646698, + "rewards/chosen": -0.6664062738418579, + "rewards/margins": 1.2390625476837158, + "rewards/rejected": -1.907812476158142, + "step": 2310 + }, + { + "epoch": 0.5960350154479918, + "grad_norm": 502.0, + "learning_rate": 2.0198249227600412e-07, + "logits/chosen": -2.5999999046325684, + "logits/rejected": -2.65625, + "logps/chosen": -284.6000061035156, + "logps/rejected": -283.3999938964844, + "loss": 0.5848, + "rewards/accuracies": 0.668315052986145, + "rewards/chosen": -0.23383788764476776, + "rewards/margins": 0.518750011920929, + "rewards/rejected": -0.7529296875, + "step": 2315 + }, + { + "epoch": 0.5973223480947477, + "grad_norm": 512.0, + "learning_rate": 2.0133882595262617e-07, + "logits/chosen": -2.4906249046325684, + "logits/rejected": -2.234375, + "logps/chosen": -329.79998779296875, + "logps/rejected": -368.0, + "loss": 0.5434, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.7178710699081421, + "rewards/margins": 0.761523425579071, + "rewards/rejected": -1.476171851158142, + "step": 2320 + }, + { + "epoch": 0.5986096807415036, + "grad_norm": 564.0, + "learning_rate": 2.0069515962924818e-07, + "logits/chosen": -2.5875000953674316, + "logits/rejected": -2.575000047683716, + "logps/chosen": -255.1999969482422, + "logps/rejected": -355.79998779296875, + "loss": 0.475, + "rewards/accuracies": 0.7629762291908264, + "rewards/chosen": -0.6182616949081421, + "rewards/margins": 1.0968749523162842, + "rewards/rejected": -1.71484375, + "step": 2325 + }, + { + "epoch": 0.5998970133882595, + "grad_norm": 1168.0, + "learning_rate": 2.0005149330587023e-07, + "logits/chosen": -2.6781249046325684, + "logits/rejected": -2.53125, + "logps/chosen": -200.35000610351562, + "logps/rejected": -202.1999969482422, + "loss": 0.5859, + "rewards/accuracies": 0.8166667222976685, + "rewards/chosen": -0.05839843675494194, + "rewards/margins": 0.701953113079071, + "rewards/rejected": -0.760937511920929, + "step": 2330 + }, + { + "epoch": 0.6011843460350155, + "grad_norm": 352.0, + "learning_rate": 1.9940782698249227e-07, + "logits/chosen": -2.6812500953674316, + "logits/rejected": -2.59375, + "logps/chosen": -309.0, + "logps/rejected": -412.0, + "loss": 0.4301, + "rewards/accuracies": 0.7549999952316284, + "rewards/chosen": -0.40642088651657104, + "rewards/margins": 1.4265625476837158, + "rewards/rejected": -1.8312499523162842, + "step": 2335 + }, + { + "epoch": 0.6024716786817713, + "grad_norm": 644.0, + "learning_rate": 1.987641606591143e-07, + "logits/chosen": -2.5406250953674316, + "logits/rejected": -2.4937500953674316, + "logps/chosen": -320.3999938964844, + "logps/rejected": -366.0, + "loss": 0.573, + "rewards/accuracies": 0.648626446723938, + "rewards/chosen": -0.39189451932907104, + "rewards/margins": 0.5874999761581421, + "rewards/rejected": -0.9789062738418579, + "step": 2340 + }, + { + "epoch": 0.6037590113285273, + "grad_norm": 496.0, + "learning_rate": 1.9812049433573635e-07, + "logits/chosen": -2.606250047683716, + "logits/rejected": -2.5062499046325684, + "logps/chosen": -300.79998779296875, + "logps/rejected": -327.29998779296875, + "loss": 0.4828, + "rewards/accuracies": 0.7611111402511597, + "rewards/chosen": -0.4917968809604645, + "rewards/margins": 0.9373779296875, + "rewards/rejected": -1.427343726158142, + "step": 2345 + }, + { + "epoch": 0.6050463439752832, + "grad_norm": 752.0, + "learning_rate": 1.9747682801235837e-07, + "logits/chosen": -2.4781250953674316, + "logits/rejected": -2.106250047683716, + "logps/chosen": -253.1999969482422, + "logps/rejected": -322.6000061035156, + "loss": 0.5336, + "rewards/accuracies": 0.6566666960716248, + "rewards/chosen": -0.708984375, + "rewards/margins": 1.1388671398162842, + "rewards/rejected": -1.851953148841858, + "step": 2350 + }, + { + "epoch": 0.6063336766220392, + "grad_norm": 528.0, + "learning_rate": 1.9683316168898044e-07, + "logits/chosen": -2.456249952316284, + "logits/rejected": -2.578125, + "logps/chosen": -317.6000061035156, + "logps/rejected": -338.79998779296875, + "loss": 0.5047, + "rewards/accuracies": 0.7191666960716248, + "rewards/chosen": -0.3267578184604645, + "rewards/margins": 0.9830077886581421, + "rewards/rejected": -1.3093750476837158, + "step": 2355 + }, + { + "epoch": 0.607621009268795, + "grad_norm": 940.0, + "learning_rate": 1.9618949536560248e-07, + "logits/chosen": -2.5093750953674316, + "logits/rejected": -2.5562500953674316, + "logps/chosen": -328.0, + "logps/rejected": -428.79998779296875, + "loss": 0.4316, + "rewards/accuracies": 0.7555769681930542, + "rewards/chosen": -1.0320312976837158, + "rewards/margins": 1.365625023841858, + "rewards/rejected": -2.395312547683716, + "step": 2360 + }, + { + "epoch": 0.608908341915551, + "grad_norm": 540.0, + "learning_rate": 1.955458290422245e-07, + "logits/chosen": -2.75, + "logits/rejected": -2.815624952316284, + "logps/chosen": -278.79998779296875, + "logps/rejected": -290.3999938964844, + "loss": 0.473, + "rewards/accuracies": 0.760217547416687, + "rewards/chosen": -0.4613281190395355, + "rewards/margins": 0.8257812261581421, + "rewards/rejected": -1.2882812023162842, + "step": 2365 + }, + { + "epoch": 0.6101956745623069, + "grad_norm": 434.0, + "learning_rate": 1.9490216271884654e-07, + "logits/chosen": -2.4000000953674316, + "logits/rejected": -2.1640625, + "logps/chosen": -288.3999938964844, + "logps/rejected": -336.0, + "loss": 0.4871, + "rewards/accuracies": 0.7503787875175476, + "rewards/chosen": -0.686328113079071, + "rewards/margins": 1.1189453601837158, + "rewards/rejected": -1.8074219226837158, + "step": 2370 + }, + { + "epoch": 0.6114830072090628, + "grad_norm": 612.0, + "learning_rate": 1.9425849639546856e-07, + "logits/chosen": -2.5687499046325684, + "logits/rejected": -2.581249952316284, + "logps/chosen": -256.0, + "logps/rejected": -310.3999938964844, + "loss": 0.6398, + "rewards/accuracies": 0.6735140085220337, + "rewards/chosen": -0.49394530057907104, + "rewards/margins": 0.6786133050918579, + "rewards/rejected": -1.174218773841858, + "step": 2375 + }, + { + "epoch": 0.6127703398558187, + "grad_norm": 388.0, + "learning_rate": 1.9361483007209063e-07, + "logits/chosen": -2.762500047683716, + "logits/rejected": -2.7906250953674316, + "logps/chosen": -311.0, + "logps/rejected": -343.79998779296875, + "loss": 0.4461, + "rewards/accuracies": 0.8088736534118652, + "rewards/chosen": -0.2637695372104645, + "rewards/margins": 0.973437488079071, + "rewards/rejected": -1.2373046875, + "step": 2380 + }, + { + "epoch": 0.6140576725025747, + "grad_norm": 310.0, + "learning_rate": 1.9297116374871267e-07, + "logits/chosen": -2.674999952316284, + "logits/rejected": -2.3843750953674316, + "logps/chosen": -275.29998779296875, + "logps/rejected": -252.0, + "loss": 0.5465, + "rewards/accuracies": 0.7522619366645813, + "rewards/chosen": -0.42451173067092896, + "rewards/margins": 0.6502929925918579, + "rewards/rejected": -1.074609398841858, + "step": 2385 + }, + { + "epoch": 0.6153450051493305, + "grad_norm": 644.0, + "learning_rate": 1.9232749742533468e-07, + "logits/chosen": -2.5, + "logits/rejected": -2.778125047683716, + "logps/chosen": -239.0, + "logps/rejected": -268.20001220703125, + "loss": 0.5281, + "rewards/accuracies": 0.6308876872062683, + "rewards/chosen": -0.2562011778354645, + "rewards/margins": 0.527148425579071, + "rewards/rejected": -0.7847656011581421, + "step": 2390 + }, + { + "epoch": 0.6166323377960865, + "grad_norm": 2416.0, + "learning_rate": 1.9168383110195673e-07, + "logits/chosen": -2.487499952316284, + "logits/rejected": -2.528125047683716, + "logps/chosen": -367.6000061035156, + "logps/rejected": -410.79998779296875, + "loss": 0.6273, + "rewards/accuracies": 0.6825000047683716, + "rewards/chosen": -0.6654297113418579, + "rewards/margins": 0.84765625, + "rewards/rejected": -1.5105469226837158, + "step": 2395 + }, + { + "epoch": 0.6179196704428425, + "grad_norm": 620.0, + "learning_rate": 1.910401647785788e-07, + "logits/chosen": -2.487499952316284, + "logits/rejected": -2.284374952316284, + "logps/chosen": -287.79998779296875, + "logps/rejected": -406.3999938964844, + "loss": 0.4527, + "rewards/accuracies": 0.7275000214576721, + "rewards/chosen": -0.781445324420929, + "rewards/margins": 1.4109375476837158, + "rewards/rejected": -2.192187547683716, + "step": 2400 + }, + { + "epoch": 0.6192070030895983, + "grad_norm": 588.0, + "learning_rate": 1.903964984552008e-07, + "logits/chosen": -2.5625, + "logits/rejected": -2.643749952316284, + "logps/chosen": -301.79998779296875, + "logps/rejected": -270.20001220703125, + "loss": 0.6309, + "rewards/accuracies": 0.5390079617500305, + "rewards/chosen": -0.6460937261581421, + "rewards/margins": 0.46137696504592896, + "rewards/rejected": -1.1066405773162842, + "step": 2405 + }, + { + "epoch": 0.6204943357363543, + "grad_norm": 636.0, + "learning_rate": 1.8975283213182286e-07, + "logits/chosen": -2.6624999046325684, + "logits/rejected": -2.8031249046325684, + "logps/chosen": -387.20001220703125, + "logps/rejected": -391.6000061035156, + "loss": 0.4285, + "rewards/accuracies": 0.8089286088943481, + "rewards/chosen": -0.4687255918979645, + "rewards/margins": 1.212499976158142, + "rewards/rejected": -1.6828124523162842, + "step": 2410 + }, + { + "epoch": 0.6217816683831102, + "grad_norm": 820.0, + "learning_rate": 1.8910916580844487e-07, + "logits/chosen": -2.628124952316284, + "logits/rejected": -2.78125, + "logps/chosen": -328.0, + "logps/rejected": -293.79998779296875, + "loss": 0.7281, + "rewards/accuracies": 0.6007539629936218, + "rewards/chosen": -0.3822265565395355, + "rewards/margins": 0.27294921875, + "rewards/rejected": -0.655468761920929, + "step": 2415 + }, + { + "epoch": 0.6230690010298661, + "grad_norm": 420.0, + "learning_rate": 1.8846549948506694e-07, + "logits/chosen": -2.674999952316284, + "logits/rejected": -2.637500047683716, + "logps/chosen": -311.20001220703125, + "logps/rejected": -347.20001220703125, + "loss": 0.4855, + "rewards/accuracies": 0.7020995616912842, + "rewards/chosen": -0.600292980670929, + "rewards/margins": 0.938281238079071, + "rewards/rejected": -1.5390625, + "step": 2420 + }, + { + "epoch": 0.624356333676622, + "grad_norm": 460.0, + "learning_rate": 1.8782183316168898e-07, + "logits/chosen": -2.628124952316284, + "logits/rejected": -2.5562500953674316, + "logps/chosen": -330.3999938964844, + "logps/rejected": -388.3999938964844, + "loss": 0.4199, + "rewards/accuracies": 0.7876191139221191, + "rewards/chosen": -0.505908191204071, + "rewards/margins": 1.1640625, + "rewards/rejected": -1.669531226158142, + "step": 2425 + }, + { + "epoch": 0.625643666323378, + "grad_norm": 668.0, + "learning_rate": 1.87178166838311e-07, + "logits/chosen": -2.590625047683716, + "logits/rejected": -2.659374952316284, + "logps/chosen": -329.6000061035156, + "logps/rejected": -355.79998779296875, + "loss": 0.5637, + "rewards/accuracies": 0.6857143044471741, + "rewards/chosen": -0.558276355266571, + "rewards/margins": 0.684765636920929, + "rewards/rejected": -1.242578148841858, + "step": 2430 + }, + { + "epoch": 0.6269309989701339, + "grad_norm": 412.0, + "learning_rate": 1.8653450051493304e-07, + "logits/chosen": -2.6624999046325684, + "logits/rejected": -2.703125, + "logps/chosen": -329.79998779296875, + "logps/rejected": -343.3999938964844, + "loss": 0.6555, + "rewards/accuracies": 0.6520237922668457, + "rewards/chosen": -0.47807615995407104, + "rewards/margins": 0.46123045682907104, + "rewards/rejected": -0.9390624761581421, + "step": 2435 + }, + { + "epoch": 0.6282183316168898, + "grad_norm": 620.0, + "learning_rate": 1.858908341915551e-07, + "logits/chosen": -2.6312499046325684, + "logits/rejected": -2.6500000953674316, + "logps/chosen": -336.6000061035156, + "logps/rejected": -399.20001220703125, + "loss": 0.6559, + "rewards/accuracies": 0.6544047594070435, + "rewards/chosen": -0.9766601324081421, + "rewards/margins": 0.704150378704071, + "rewards/rejected": -1.6828124523162842, + "step": 2440 + }, + { + "epoch": 0.6295056642636457, + "grad_norm": 506.0, + "learning_rate": 1.8524716786817713e-07, + "logits/chosen": -2.5250000953674316, + "logits/rejected": -2.659374952316284, + "logps/chosen": -247.0, + "logps/rejected": -265.79998779296875, + "loss": 0.6305, + "rewards/accuracies": 0.4751947820186615, + "rewards/chosen": -0.385498046875, + "rewards/margins": 0.3514648377895355, + "rewards/rejected": -0.7376953363418579, + "step": 2445 + }, + { + "epoch": 0.6307929969104017, + "grad_norm": 548.0, + "learning_rate": 1.8460350154479917e-07, + "logits/chosen": -2.7093749046325684, + "logits/rejected": -2.643749952316284, + "logps/chosen": -320.79998779296875, + "logps/rejected": -328.20001220703125, + "loss": 0.5312, + "rewards/accuracies": 0.7666667103767395, + "rewards/chosen": -0.40540528297424316, + "rewards/margins": 0.935351550579071, + "rewards/rejected": -1.3425781726837158, + "step": 2450 + }, + { + "epoch": 0.6320803295571575, + "grad_norm": 470.0, + "learning_rate": 1.8395983522142119e-07, + "logits/chosen": -2.7281250953674316, + "logits/rejected": -2.684375047683716, + "logps/chosen": -347.6000061035156, + "logps/rejected": -411.20001220703125, + "loss": 0.5508, + "rewards/accuracies": 0.6508333683013916, + "rewards/chosen": -0.629687488079071, + "rewards/margins": 0.986132800579071, + "rewards/rejected": -1.614843726158142, + "step": 2455 + }, + { + "epoch": 0.6333676622039135, + "grad_norm": 568.0, + "learning_rate": 1.8331616889804326e-07, + "logits/chosen": -2.5625, + "logits/rejected": -2.581249952316284, + "logps/chosen": -348.79998779296875, + "logps/rejected": -416.0, + "loss": 0.4361, + "rewards/accuracies": 0.7737878561019897, + "rewards/chosen": -0.860546886920929, + "rewards/margins": 1.204687476158142, + "rewards/rejected": -2.067187547683716, + "step": 2460 + }, + { + "epoch": 0.6346549948506695, + "grad_norm": 524.0, + "learning_rate": 1.826725025746653e-07, + "logits/chosen": -2.7718749046325684, + "logits/rejected": -2.7281250953674316, + "logps/chosen": -366.3999938964844, + "logps/rejected": -415.20001220703125, + "loss": 0.4969, + "rewards/accuracies": 0.7235257029533386, + "rewards/chosen": -0.4583984315395355, + "rewards/margins": 0.958203136920929, + "rewards/rejected": -1.4171874523162842, + "step": 2465 + }, + { + "epoch": 0.6359423274974253, + "grad_norm": 422.0, + "learning_rate": 1.8202883625128731e-07, + "logits/chosen": -2.6312499046325684, + "logits/rejected": -2.659374952316284, + "logps/chosen": -257.3999938964844, + "logps/rejected": -313.79998779296875, + "loss": 0.4652, + "rewards/accuracies": 0.7094444632530212, + "rewards/chosen": -0.1099853515625, + "rewards/margins": 0.897265613079071, + "rewards/rejected": -1.008203148841858, + "step": 2470 + }, + { + "epoch": 0.6372296601441813, + "grad_norm": 486.0, + "learning_rate": 1.8138516992790936e-07, + "logits/chosen": -2.456249952316284, + "logits/rejected": -2.5718750953674316, + "logps/chosen": -342.3999938964844, + "logps/rejected": -386.79998779296875, + "loss": 0.5363, + "rewards/accuracies": 0.7583333849906921, + "rewards/chosen": -0.8062499761581421, + "rewards/margins": 0.9798828363418579, + "rewards/rejected": -1.785546898841858, + "step": 2475 + }, + { + "epoch": 0.6385169927909372, + "grad_norm": 676.0, + "learning_rate": 1.8074150360453143e-07, + "logits/chosen": -2.734375, + "logits/rejected": -2.75, + "logps/chosen": -301.3999938964844, + "logps/rejected": -384.3999938964844, + "loss": 0.432, + "rewards/accuracies": 0.7925000190734863, + "rewards/chosen": -0.447265625, + "rewards/margins": 1.1476562023162842, + "rewards/rejected": -1.59375, + "step": 2480 + }, + { + "epoch": 0.6398043254376931, + "grad_norm": 348.0, + "learning_rate": 1.8009783728115344e-07, + "logits/chosen": -2.668750047683716, + "logits/rejected": -2.575000047683716, + "logps/chosen": -340.20001220703125, + "logps/rejected": -393.20001220703125, + "loss": 0.4898, + "rewards/accuracies": 0.7024999856948853, + "rewards/chosen": -0.9125000238418579, + "rewards/margins": 0.9085937738418579, + "rewards/rejected": -1.822851538658142, + "step": 2485 + }, + { + "epoch": 0.641091658084449, + "grad_norm": 440.0, + "learning_rate": 1.7945417095777549e-07, + "logits/chosen": -2.515625, + "logits/rejected": -2.387500047683716, + "logps/chosen": -262.6000061035156, + "logps/rejected": -353.79998779296875, + "loss": 0.4473, + "rewards/accuracies": 0.7603572010993958, + "rewards/chosen": -0.6494140625, + "rewards/margins": 1.224609375, + "rewards/rejected": -1.872656226158142, + "step": 2490 + }, + { + "epoch": 0.642378990731205, + "grad_norm": 560.0, + "learning_rate": 1.788105046343975e-07, + "logits/chosen": -2.5374999046325684, + "logits/rejected": -2.481250047683716, + "logps/chosen": -253.6999969482422, + "logps/rejected": -269.1000061035156, + "loss": 0.6145, + "rewards/accuracies": 0.5552381277084351, + "rewards/chosen": -0.44414061307907104, + "rewards/margins": 0.59033203125, + "rewards/rejected": -1.0369141101837158, + "step": 2495 + }, + { + "epoch": 0.6436663233779608, + "grad_norm": 498.0, + "learning_rate": 1.7816683831101954e-07, + "logits/chosen": -2.659374952316284, + "logits/rejected": -2.59375, + "logps/chosen": -367.6000061035156, + "logps/rejected": -408.79998779296875, + "loss": 0.4594, + "rewards/accuracies": 0.8128571510314941, + "rewards/chosen": -0.8662109375, + "rewards/margins": 1.024999976158142, + "rewards/rejected": -1.890625, + "step": 2500 + }, + { + "epoch": 0.6449536560247168, + "grad_norm": 1496.0, + "learning_rate": 1.7752317198764161e-07, + "logits/chosen": -2.7249999046325684, + "logits/rejected": -2.7593750953674316, + "logps/chosen": -305.79998779296875, + "logps/rejected": -292.0, + "loss": 0.6332, + "rewards/accuracies": 0.6858333349227905, + "rewards/chosen": -0.47099608182907104, + "rewards/margins": 0.48320311307907104, + "rewards/rejected": -0.953906238079071, + "step": 2505 + }, + { + "epoch": 0.6462409886714727, + "grad_norm": 438.0, + "learning_rate": 1.7687950566426363e-07, + "logits/chosen": -2.4375, + "logits/rejected": -2.5843749046325684, + "logps/chosen": -275.0, + "logps/rejected": -265.3500061035156, + "loss": 0.6484, + "rewards/accuracies": 0.7041667103767395, + "rewards/chosen": -0.17452391982078552, + "rewards/margins": 0.3238281309604645, + "rewards/rejected": -0.49882811307907104, + "step": 2510 + }, + { + "epoch": 0.6475283213182287, + "grad_norm": 780.0, + "learning_rate": 1.7623583934088567e-07, + "logits/chosen": -2.65625, + "logits/rejected": -2.7562499046325684, + "logps/chosen": -373.20001220703125, + "logps/rejected": -324.79998779296875, + "loss": 0.6016, + "rewards/accuracies": 0.6493590474128723, + "rewards/chosen": -0.32832032442092896, + "rewards/margins": 0.4970703125, + "rewards/rejected": -0.824999988079071, + "step": 2515 + }, + { + "epoch": 0.6488156539649845, + "grad_norm": 384.0, + "learning_rate": 1.7559217301750771e-07, + "logits/chosen": -2.765625, + "logits/rejected": -2.7249999046325684, + "logps/chosen": -349.20001220703125, + "logps/rejected": -368.3999938964844, + "loss": 0.4738, + "rewards/accuracies": 0.7400000691413879, + "rewards/chosen": -0.2660156190395355, + "rewards/margins": 0.866406261920929, + "rewards/rejected": -1.131250023841858, + "step": 2520 + }, + { + "epoch": 0.6501029866117405, + "grad_norm": 604.0, + "learning_rate": 1.7494850669412976e-07, + "logits/chosen": -2.5687499046325684, + "logits/rejected": -2.7093749046325684, + "logps/chosen": -367.6000061035156, + "logps/rejected": -407.6000061035156, + "loss": 0.4707, + "rewards/accuracies": 0.7524999976158142, + "rewards/chosen": -0.6107422113418579, + "rewards/margins": 0.995312511920929, + "rewards/rejected": -1.60546875, + "step": 2525 + }, + { + "epoch": 0.6513903192584964, + "grad_norm": 712.0, + "learning_rate": 1.743048403707518e-07, + "logits/chosen": -2.609375, + "logits/rejected": -2.5843749046325684, + "logps/chosen": -340.20001220703125, + "logps/rejected": -387.79998779296875, + "loss": 0.4271, + "rewards/accuracies": 0.7933333516120911, + "rewards/chosen": -0.44501954317092896, + "rewards/margins": 1.253515601158142, + "rewards/rejected": -1.703515648841858, + "step": 2530 + }, + { + "epoch": 0.6526776519052523, + "grad_norm": 876.0, + "learning_rate": 1.7366117404737382e-07, + "logits/chosen": -2.609375, + "logits/rejected": -2.731250047683716, + "logps/chosen": -329.20001220703125, + "logps/rejected": -230.0, + "loss": 0.5711, + "rewards/accuracies": 0.6717948913574219, + "rewards/chosen": -0.2582031190395355, + "rewards/margins": 0.599414050579071, + "rewards/rejected": -0.8578125238418579, + "step": 2535 + }, + { + "epoch": 0.6539649845520082, + "grad_norm": 540.0, + "learning_rate": 1.7301750772399586e-07, + "logits/chosen": -2.6343750953674316, + "logits/rejected": -2.575000047683716, + "logps/chosen": -342.20001220703125, + "logps/rejected": -399.3999938964844, + "loss": 0.4844, + "rewards/accuracies": 0.7217949032783508, + "rewards/chosen": -1.06640625, + "rewards/margins": 1.1906249523162842, + "rewards/rejected": -2.2578125, + "step": 2540 + }, + { + "epoch": 0.6552523171987642, + "grad_norm": 520.0, + "learning_rate": 1.7237384140061793e-07, + "logits/chosen": -2.549999952316284, + "logits/rejected": -2.578125, + "logps/chosen": -270.3999938964844, + "logps/rejected": -324.3999938964844, + "loss": 0.6379, + "rewards/accuracies": 0.7090476751327515, + "rewards/chosen": -0.910937488079071, + "rewards/margins": 0.893750011920929, + "rewards/rejected": -1.8054687976837158, + "step": 2545 + }, + { + "epoch": 0.65653964984552, + "grad_norm": 764.0, + "learning_rate": 1.7173017507723994e-07, + "logits/chosen": -2.543750047683716, + "logits/rejected": -2.621875047683716, + "logps/chosen": -303.20001220703125, + "logps/rejected": -378.79998779296875, + "loss": 0.4953, + "rewards/accuracies": 0.7423809766769409, + "rewards/chosen": -0.7035156488418579, + "rewards/margins": 1.3125, + "rewards/rejected": -2.0146484375, + "step": 2550 + }, + { + "epoch": 0.657826982492276, + "grad_norm": 524.0, + "learning_rate": 1.71086508753862e-07, + "logits/chosen": -2.4375, + "logits/rejected": -2.299999952316284, + "logps/chosen": -302.20001220703125, + "logps/rejected": -312.20001220703125, + "loss": 0.4703, + "rewards/accuracies": 0.6885714530944824, + "rewards/chosen": -0.573437511920929, + "rewards/margins": 1.148828148841858, + "rewards/rejected": -1.7234375476837158, + "step": 2555 + }, + { + "epoch": 0.659114315139032, + "grad_norm": 628.0, + "learning_rate": 1.7044284243048403e-07, + "logits/chosen": -2.653125047683716, + "logits/rejected": -2.668750047683716, + "logps/chosen": -261.3999938964844, + "logps/rejected": -298.3999938964844, + "loss": 0.6094, + "rewards/accuracies": 0.6992857456207275, + "rewards/chosen": -0.4917968809604645, + "rewards/margins": 0.620898425579071, + "rewards/rejected": -1.1129882335662842, + "step": 2560 + }, + { + "epoch": 0.6604016477857878, + "grad_norm": 460.0, + "learning_rate": 1.6979917610710607e-07, + "logits/chosen": -2.549999952316284, + "logits/rejected": -2.6187500953674316, + "logps/chosen": -243.10000610351562, + "logps/rejected": -337.20001220703125, + "loss": 0.4445, + "rewards/accuracies": 0.721666693687439, + "rewards/chosen": -0.2701171934604645, + "rewards/margins": 1.0271484851837158, + "rewards/rejected": -1.2980468273162842, + "step": 2565 + }, + { + "epoch": 0.6616889804325438, + "grad_norm": 512.0, + "learning_rate": 1.6915550978372812e-07, + "logits/chosen": -2.796875, + "logits/rejected": -2.8125, + "logps/chosen": -381.6000061035156, + "logps/rejected": -334.79998779296875, + "loss": 0.5586, + "rewards/accuracies": 0.6916667222976685, + "rewards/chosen": -0.29179686307907104, + "rewards/margins": 0.5550781488418579, + "rewards/rejected": -0.846484363079071, + "step": 2570 + }, + { + "epoch": 0.6629763130792997, + "grad_norm": 360.0, + "learning_rate": 1.6851184346035013e-07, + "logits/chosen": -2.7718749046325684, + "logits/rejected": -2.721874952316284, + "logps/chosen": -350.0, + "logps/rejected": -305.79998779296875, + "loss": 0.5262, + "rewards/accuracies": 0.6966667175292969, + "rewards/chosen": -0.26817625761032104, + "rewards/margins": 0.747265636920929, + "rewards/rejected": -1.0148437023162842, + "step": 2575 + }, + { + "epoch": 0.6642636457260556, + "grad_norm": 454.0, + "learning_rate": 1.6786817713697217e-07, + "logits/chosen": -2.5843749046325684, + "logits/rejected": -2.609375, + "logps/chosen": -381.6000061035156, + "logps/rejected": -498.3999938964844, + "loss": 0.4002, + "rewards/accuracies": 0.8207143545150757, + "rewards/chosen": -0.854296863079071, + "rewards/margins": 1.2693359851837158, + "rewards/rejected": -2.1226563453674316, + "step": 2580 + }, + { + "epoch": 0.6655509783728115, + "grad_norm": 306.0, + "learning_rate": 1.6722451081359424e-07, + "logits/chosen": -2.784374952316284, + "logits/rejected": -2.7125000953674316, + "logps/chosen": -271.6000061035156, + "logps/rejected": -370.6000061035156, + "loss": 0.4551, + "rewards/accuracies": 0.7600000500679016, + "rewards/chosen": -0.5296875238418579, + "rewards/margins": 1.1015625, + "rewards/rejected": -1.6300780773162842, + "step": 2585 + }, + { + "epoch": 0.6668383110195675, + "grad_norm": 504.0, + "learning_rate": 1.6658084449021626e-07, + "logits/chosen": -2.71875, + "logits/rejected": -2.6937499046325684, + "logps/chosen": -258.20001220703125, + "logps/rejected": -256.20001220703125, + "loss": 0.5152, + "rewards/accuracies": 0.7171001434326172, + "rewards/chosen": -0.30229490995407104, + "rewards/margins": 0.842578113079071, + "rewards/rejected": -1.142968773841858, + "step": 2590 + }, + { + "epoch": 0.6681256436663234, + "grad_norm": 600.0, + "learning_rate": 1.659371781668383e-07, + "logits/chosen": -2.643749952316284, + "logits/rejected": -2.6781249046325684, + "logps/chosen": -291.0, + "logps/rejected": -361.20001220703125, + "loss": 0.5074, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.4598144590854645, + "rewards/margins": 1.034765601158142, + "rewards/rejected": -1.4968750476837158, + "step": 2595 + }, + { + "epoch": 0.6694129763130793, + "grad_norm": 292.0, + "learning_rate": 1.6529351184346034e-07, + "logits/chosen": -2.684375047683716, + "logits/rejected": -2.690624952316284, + "logps/chosen": -339.20001220703125, + "logps/rejected": -369.6000061035156, + "loss": 0.3988, + "rewards/accuracies": 0.766883134841919, + "rewards/chosen": -0.46904295682907104, + "rewards/margins": 1.2742187976837158, + "rewards/rejected": -1.744531273841858, + "step": 2600 + }, + { + "epoch": 0.6707003089598352, + "grad_norm": 398.0, + "learning_rate": 1.646498455200824e-07, + "logits/chosen": -2.5250000953674316, + "logits/rejected": -2.581249952316284, + "logps/chosen": -279.6000061035156, + "logps/rejected": -324.0, + "loss": 0.4598, + "rewards/accuracies": 0.7406060695648193, + "rewards/chosen": -0.4970703125, + "rewards/margins": 0.942187488079071, + "rewards/rejected": -1.439062476158142, + "step": 2605 + }, + { + "epoch": 0.6719876416065912, + "grad_norm": 422.0, + "learning_rate": 1.6400617919670443e-07, + "logits/chosen": -2.643749952316284, + "logits/rejected": NaN, + "logps/chosen": -287.6000061035156, + "logps/rejected": -292.0, + "loss": 0.4648, + "rewards/accuracies": 0.6482142806053162, + "rewards/chosen": -0.20761719346046448, + "rewards/margins": 0.904296875, + "rewards/rejected": -1.1116211414337158, + "step": 2610 + }, + { + "epoch": 0.673274974253347, + "grad_norm": 502.0, + "learning_rate": 1.6336251287332645e-07, + "logits/chosen": -2.4156250953674316, + "logits/rejected": -2.575000047683716, + "logps/chosen": -362.3999938964844, + "logps/rejected": -395.20001220703125, + "loss": 0.5875, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.930859386920929, + "rewards/margins": 0.78955078125, + "rewards/rejected": -1.72265625, + "step": 2615 + }, + { + "epoch": 0.674562306900103, + "grad_norm": 498.0, + "learning_rate": 1.627188465499485e-07, + "logits/chosen": -2.546875, + "logits/rejected": -2.637500047683716, + "logps/chosen": -290.79998779296875, + "logps/rejected": -381.20001220703125, + "loss": 0.4744, + "rewards/accuracies": 0.7271677851676941, + "rewards/chosen": -0.34465330839157104, + "rewards/margins": 0.9742187261581421, + "rewards/rejected": -1.318750023841858, + "step": 2620 + }, + { + "epoch": 0.675849639546859, + "grad_norm": 440.0, + "learning_rate": 1.6207518022657056e-07, + "logits/chosen": -2.65625, + "logits/rejected": -2.2466063499450684, + "logps/chosen": -243.0, + "logps/rejected": -217.25, + "loss": 0.5832, + "rewards/accuracies": 0.6914652585983276, + "rewards/chosen": -0.203857421875, + "rewards/margins": 0.4505371153354645, + "rewards/rejected": -0.6552734375, + "step": 2625 + }, + { + "epoch": 0.6771369721936148, + "grad_norm": 406.0, + "learning_rate": 1.6143151390319257e-07, + "logits/chosen": -2.465625047683716, + "logits/rejected": -2.503124952316284, + "logps/chosen": -294.3999938964844, + "logps/rejected": -362.0, + "loss": 0.4449, + "rewards/accuracies": 0.7269231081008911, + "rewards/chosen": -0.813183605670929, + "rewards/margins": 1.1403319835662842, + "rewards/rejected": -1.94873046875, + "step": 2630 + }, + { + "epoch": 0.6784243048403708, + "grad_norm": 532.0, + "learning_rate": 1.6078784757981462e-07, + "logits/chosen": -2.5250000953674316, + "logits/rejected": -2.496875047683716, + "logps/chosen": -261.20001220703125, + "logps/rejected": -243.60000610351562, + "loss": 0.5191, + "rewards/accuracies": 0.6843290328979492, + "rewards/chosen": -0.3755859434604645, + "rewards/margins": 0.8199218511581421, + "rewards/rejected": -1.1953125, + "step": 2635 + }, + { + "epoch": 0.6797116374871267, + "grad_norm": 382.0, + "learning_rate": 1.6014418125643666e-07, + "logits/chosen": -2.659374952316284, + "logits/rejected": -2.765625, + "logps/chosen": -289.3999938964844, + "logps/rejected": -338.79998779296875, + "loss": 0.4375, + "rewards/accuracies": 0.7491666674613953, + "rewards/chosen": -0.1357421875, + "rewards/margins": 1.1472656726837158, + "rewards/rejected": -1.2824218273162842, + "step": 2640 + }, + { + "epoch": 0.6809989701338826, + "grad_norm": 676.0, + "learning_rate": 1.5950051493305868e-07, + "logits/chosen": -2.546875, + "logits/rejected": -2.640625, + "logps/chosen": -206.14999389648438, + "logps/rejected": -226.1999969482422, + "loss": 0.5305, + "rewards/accuracies": 0.6428030729293823, + "rewards/chosen": -0.011914062313735485, + "rewards/margins": 0.697265625, + "rewards/rejected": -0.7099609375, + "step": 2645 + }, + { + "epoch": 0.6822863027806385, + "grad_norm": 492.0, + "learning_rate": 1.5885684860968075e-07, + "logits/chosen": -2.6624999046325684, + "logits/rejected": -2.6343750953674316, + "logps/chosen": -324.3999938964844, + "logps/rejected": -370.79998779296875, + "loss": 0.4645, + "rewards/accuracies": 0.7074999809265137, + "rewards/chosen": -0.7906249761581421, + "rewards/margins": 1.0382812023162842, + "rewards/rejected": -1.829687476158142, + "step": 2650 + }, + { + "epoch": 0.6835736354273945, + "grad_norm": 442.0, + "learning_rate": 1.5821318228630276e-07, + "logits/chosen": -2.5875000953674316, + "logits/rejected": -2.575000047683716, + "logps/chosen": -320.0, + "logps/rejected": -413.6000061035156, + "loss": 0.4059, + "rewards/accuracies": 0.7442857623100281, + "rewards/chosen": -0.704296886920929, + "rewards/margins": 1.4093749523162842, + "rewards/rejected": -2.1109375953674316, + "step": 2655 + }, + { + "epoch": 0.6848609680741503, + "grad_norm": 676.0, + "learning_rate": 1.575695159629248e-07, + "logits/chosen": -2.534374952316284, + "logits/rejected": -2.4749999046325684, + "logps/chosen": -395.6000061035156, + "logps/rejected": -370.79998779296875, + "loss": 0.548, + "rewards/accuracies": 0.6666666865348816, + "rewards/chosen": -1.1515624523162842, + "rewards/margins": 1.005468726158142, + "rewards/rejected": -2.15625, + "step": 2660 + }, + { + "epoch": 0.6861483007209063, + "grad_norm": 572.0, + "learning_rate": 1.5692584963954685e-07, + "logits/chosen": -2.700000047683716, + "logits/rejected": -2.703125, + "logps/chosen": -262.6000061035156, + "logps/rejected": -300.6000061035156, + "loss": 0.518, + "rewards/accuracies": 0.6846795082092285, + "rewards/chosen": -0.4012695252895355, + "rewards/margins": 0.842968761920929, + "rewards/rejected": -1.2421875, + "step": 2665 + }, + { + "epoch": 0.6874356333676622, + "grad_norm": 556.0, + "learning_rate": 1.562821833161689e-07, + "logits/chosen": -2.609375, + "logits/rejected": -2.746875047683716, + "logps/chosen": -308.79998779296875, + "logps/rejected": -344.3999938964844, + "loss": 0.5754, + "rewards/accuracies": 0.6721212267875671, + "rewards/chosen": -0.6877075433731079, + "rewards/margins": 0.662109375, + "rewards/rejected": -1.350000023841858, + "step": 2670 + }, + { + "epoch": 0.6887229660144182, + "grad_norm": 1008.0, + "learning_rate": 1.5563851699279093e-07, + "logits/chosen": -2.715625047683716, + "logits/rejected": -2.690624952316284, + "logps/chosen": -285.20001220703125, + "logps/rejected": -325.79998779296875, + "loss": 0.5402, + "rewards/accuracies": 0.6821212768554688, + "rewards/chosen": -0.40312498807907104, + "rewards/margins": 0.890625, + "rewards/rejected": -1.29443359375, + "step": 2675 + }, + { + "epoch": 0.690010298661174, + "grad_norm": 584.0, + "learning_rate": 1.5499485066941297e-07, + "logits/chosen": -2.515625, + "logits/rejected": -2.53125, + "logps/chosen": -352.79998779296875, + "logps/rejected": -461.6000061035156, + "loss": 0.3934, + "rewards/accuracies": 0.8154762387275696, + "rewards/chosen": -0.8915039300918579, + "rewards/margins": 1.572656273841858, + "rewards/rejected": -2.464062452316284, + "step": 2680 + }, + { + "epoch": 0.69129763130793, + "grad_norm": 398.0, + "learning_rate": 1.54351184346035e-07, + "logits/chosen": -2.590625047683716, + "logits/rejected": -2.5328125953674316, + "logps/chosen": -290.0, + "logps/rejected": -361.6000061035156, + "loss": 0.4797, + "rewards/accuracies": 0.7613204121589661, + "rewards/chosen": -0.43603515625, + "rewards/margins": 1.0773437023162842, + "rewards/rejected": -1.5134766101837158, + "step": 2685 + }, + { + "epoch": 0.6925849639546859, + "grad_norm": 456.0, + "learning_rate": 1.5370751802265706e-07, + "logits/chosen": -2.5250000953674316, + "logits/rejected": -2.4937500953674316, + "logps/chosen": -323.79998779296875, + "logps/rejected": -380.0, + "loss": 0.4152, + "rewards/accuracies": 0.8416666984558105, + "rewards/chosen": -0.533154308795929, + "rewards/margins": 1.2421875, + "rewards/rejected": -1.7765624523162842, + "step": 2690 + }, + { + "epoch": 0.6938722966014418, + "grad_norm": 728.0, + "learning_rate": 1.5306385169927908e-07, + "logits/chosen": -2.606250047683716, + "logits/rejected": -2.643749952316284, + "logps/chosen": -335.6000061035156, + "logps/rejected": -311.3999938964844, + "loss": 0.5668, + "rewards/accuracies": 0.7124541997909546, + "rewards/chosen": -0.545703113079071, + "rewards/margins": 0.7904297113418579, + "rewards/rejected": -1.3367187976837158, + "step": 2695 + }, + { + "epoch": 0.6951596292481977, + "grad_norm": 434.0, + "learning_rate": 1.5242018537590112e-07, + "logits/chosen": -2.5625, + "logits/rejected": -2.021484375, + "logps/chosen": -286.79998779296875, + "logps/rejected": -291.20001220703125, + "loss": 0.5785, + "rewards/accuracies": 0.6166300773620605, + "rewards/chosen": -0.2983154356479645, + "rewards/margins": 0.488525390625, + "rewards/rejected": -0.7875000238418579, + "step": 2700 + }, + { + "epoch": 0.6964469618949537, + "grad_norm": 560.0, + "learning_rate": 1.5177651905252316e-07, + "logits/chosen": -2.5859375, + "logits/rejected": -2.7718749046325684, + "logps/chosen": -319.6000061035156, + "logps/rejected": -306.20001220703125, + "loss": 0.5379, + "rewards/accuracies": 0.7296428680419922, + "rewards/chosen": -0.23342284560203552, + "rewards/margins": 0.6304687261581421, + "rewards/rejected": -0.863085925579071, + "step": 2705 + }, + { + "epoch": 0.6977342945417095, + "grad_norm": 552.0, + "learning_rate": 1.511328527291452e-07, + "logits/chosen": -2.5531249046325684, + "logits/rejected": -2.609375, + "logps/chosen": -312.6000061035156, + "logps/rejected": -371.6000061035156, + "loss": 0.4625, + "rewards/accuracies": 0.7514743804931641, + "rewards/chosen": -0.587890625, + "rewards/margins": 0.997851550579071, + "rewards/rejected": -1.58984375, + "step": 2710 + }, + { + "epoch": 0.6990216271884655, + "grad_norm": 470.0, + "learning_rate": 1.5048918640576725e-07, + "logits/chosen": -2.575000047683716, + "logits/rejected": -2.4390625953674316, + "logps/chosen": -319.20001220703125, + "logps/rejected": -406.0, + "loss": 0.4324, + "rewards/accuracies": 0.753095269203186, + "rewards/chosen": -0.891406238079071, + "rewards/margins": 1.235937476158142, + "rewards/rejected": -2.128124952316284, + "step": 2715 + }, + { + "epoch": 0.7003089598352215, + "grad_norm": 644.0, + "learning_rate": 1.498455200823893e-07, + "logits/chosen": -2.606250047683716, + "logits/rejected": -2.65625, + "logps/chosen": -345.6000061035156, + "logps/rejected": -370.20001220703125, + "loss": 0.4957, + "rewards/accuracies": 0.7520238161087036, + "rewards/chosen": -0.4818115234375, + "rewards/margins": 0.9750000238418579, + "rewards/rejected": -1.453125, + "step": 2720 + }, + { + "epoch": 0.7015962924819773, + "grad_norm": 376.0, + "learning_rate": 1.492018537590113e-07, + "logits/chosen": -2.674999952316284, + "logits/rejected": -2.6812500953674316, + "logps/chosen": -324.79998779296875, + "logps/rejected": -329.3999938964844, + "loss": 0.4383, + "rewards/accuracies": 0.7753571271896362, + "rewards/chosen": -0.4176269471645355, + "rewards/margins": 1.07421875, + "rewards/rejected": -1.4921875, + "step": 2725 + }, + { + "epoch": 0.7028836251287333, + "grad_norm": 502.0, + "learning_rate": 1.4855818743563338e-07, + "logits/chosen": -2.6500000953674316, + "logits/rejected": -2.8125, + "logps/chosen": -302.6000061035156, + "logps/rejected": -330.0, + "loss": 0.5152, + "rewards/accuracies": 0.7219047546386719, + "rewards/chosen": -0.20869140326976776, + "rewards/margins": 0.775390625, + "rewards/rejected": -0.984375, + "step": 2730 + }, + { + "epoch": 0.7041709577754892, + "grad_norm": 692.0, + "learning_rate": 1.4791452111225542e-07, + "logits/chosen": -2.612499952316284, + "logits/rejected": -2.6468749046325684, + "logps/chosen": -304.20001220703125, + "logps/rejected": -361.20001220703125, + "loss": 0.5137, + "rewards/accuracies": 0.7051190137863159, + "rewards/chosen": -0.65087890625, + "rewards/margins": 1.038671851158142, + "rewards/rejected": -1.6902344226837158, + "step": 2735 + }, + { + "epoch": 0.7054582904222451, + "grad_norm": 832.0, + "learning_rate": 1.4727085478887743e-07, + "logits/chosen": -2.6468749046325684, + "logits/rejected": -2.7406249046325684, + "logps/chosen": -299.6000061035156, + "logps/rejected": -308.79998779296875, + "loss": 0.5656, + "rewards/accuracies": 0.7052381634712219, + "rewards/chosen": -0.543383777141571, + "rewards/margins": 0.815234363079071, + "rewards/rejected": -1.357812523841858, + "step": 2740 + }, + { + "epoch": 0.706745623069001, + "grad_norm": 600.0, + "learning_rate": 1.4662718846549948e-07, + "logits/chosen": -2.75, + "logits/rejected": -2.609375, + "logps/chosen": -308.0, + "logps/rejected": -349.6000061035156, + "loss": 0.5938, + "rewards/accuracies": 0.6535714268684387, + "rewards/chosen": -0.6570495367050171, + "rewards/margins": 0.6820312738418579, + "rewards/rejected": -1.3390624523162842, + "step": 2745 + }, + { + "epoch": 0.708032955715757, + "grad_norm": 548.0, + "learning_rate": 1.4598352214212152e-07, + "logits/chosen": -2.7093749046325684, + "logits/rejected": -2.456249952316284, + "logps/chosen": -319.20001220703125, + "logps/rejected": -333.79998779296875, + "loss": 0.5672, + "rewards/accuracies": 0.6595237851142883, + "rewards/chosen": -0.29765623807907104, + "rewards/margins": 0.45703125, + "rewards/rejected": -0.753125011920929, + "step": 2750 + }, + { + "epoch": 0.7093202883625128, + "grad_norm": 358.0, + "learning_rate": 1.4533985581874356e-07, + "logits/chosen": -2.684375047683716, + "logits/rejected": -2.856250047683716, + "logps/chosen": -291.79998779296875, + "logps/rejected": -300.0, + "loss": 0.5402, + "rewards/accuracies": 0.681538462638855, + "rewards/chosen": -0.18857422471046448, + "rewards/margins": 0.585156261920929, + "rewards/rejected": -0.7734375, + "step": 2755 + }, + { + "epoch": 0.7106076210092688, + "grad_norm": 412.0, + "learning_rate": 1.446961894953656e-07, + "logits/chosen": -2.465625047683716, + "logits/rejected": -2.546875, + "logps/chosen": -259.20001220703125, + "logps/rejected": -331.0, + "loss": 0.4965, + "rewards/accuracies": 0.8518182039260864, + "rewards/chosen": -0.6285156011581421, + "rewards/margins": 1.044531226158142, + "rewards/rejected": -1.670312523841858, + "step": 2760 + }, + { + "epoch": 0.7118949536560247, + "grad_norm": 1208.0, + "learning_rate": 1.4405252317198762e-07, + "logits/chosen": -2.4671874046325684, + "logits/rejected": -2.2632813453674316, + "logps/chosen": -225.5, + "logps/rejected": -280.5, + "loss": 0.6016, + "rewards/accuracies": 0.6469047665596008, + "rewards/chosen": -0.556933581829071, + "rewards/margins": 0.5517578125, + "rewards/rejected": -1.1085937023162842, + "step": 2765 + }, + { + "epoch": 0.7131822863027807, + "grad_norm": 366.0, + "learning_rate": 1.434088568486097e-07, + "logits/chosen": -2.609375, + "logits/rejected": -2.5625, + "logps/chosen": -319.79998779296875, + "logps/rejected": -378.79998779296875, + "loss": 0.4715, + "rewards/accuracies": 0.7780953049659729, + "rewards/chosen": -0.40410155057907104, + "rewards/margins": 0.905468761920929, + "rewards/rejected": -1.310937523841858, + "step": 2770 + }, + { + "epoch": 0.7144696189495365, + "grad_norm": 604.0, + "learning_rate": 1.4276519052523173e-07, + "logits/chosen": -2.5218749046325684, + "logits/rejected": -2.8187499046325684, + "logps/chosen": -247.60000610351562, + "logps/rejected": -266.6000061035156, + "loss": 0.5914, + "rewards/accuracies": 0.7095163464546204, + "rewards/chosen": -0.51171875, + "rewards/margins": 0.5975586175918579, + "rewards/rejected": -1.1105468273162842, + "step": 2775 + }, + { + "epoch": 0.7157569515962925, + "grad_norm": 294.0, + "learning_rate": 1.4212152420185375e-07, + "logits/chosen": -2.6875, + "logits/rejected": -2.65625, + "logps/chosen": -334.3999938964844, + "logps/rejected": -345.6000061035156, + "loss": 0.5113, + "rewards/accuracies": 0.7379761934280396, + "rewards/chosen": -0.547656238079071, + "rewards/margins": 0.8343750238418579, + "rewards/rejected": -1.3828125, + "step": 2780 + }, + { + "epoch": 0.7170442842430484, + "grad_norm": 596.0, + "learning_rate": 1.414778578784758e-07, + "logits/chosen": -2.559375047683716, + "logits/rejected": -2.293750047683716, + "logps/chosen": -256.0, + "logps/rejected": -319.3999938964844, + "loss": 0.5566, + "rewards/accuracies": 0.6183333396911621, + "rewards/chosen": -0.6268554925918579, + "rewards/margins": 0.6460937261581421, + "rewards/rejected": -1.2734375, + "step": 2785 + }, + { + "epoch": 0.7183316168898043, + "grad_norm": 223.0, + "learning_rate": 1.408341915550978e-07, + "logits/chosen": -2.6031250953674316, + "logits/rejected": -2.65625, + "logps/chosen": -327.6000061035156, + "logps/rejected": -380.3999938964844, + "loss": 0.4355, + "rewards/accuracies": 0.8026922941207886, + "rewards/chosen": -0.41650390625, + "rewards/margins": 1.2355468273162842, + "rewards/rejected": -1.651953101158142, + "step": 2790 + }, + { + "epoch": 0.7196189495365602, + "grad_norm": 540.0, + "learning_rate": 1.4019052523171988e-07, + "logits/chosen": -2.487499952316284, + "logits/rejected": -2.4375, + "logps/chosen": -312.79998779296875, + "logps/rejected": -402.3999938964844, + "loss": 0.4715, + "rewards/accuracies": 0.7491666674613953, + "rewards/chosen": -0.7191406488418579, + "rewards/margins": 1.244531273841858, + "rewards/rejected": -1.9640624523162842, + "step": 2795 + }, + { + "epoch": 0.7209062821833162, + "grad_norm": 350.0, + "learning_rate": 1.3954685890834192e-07, + "logits/chosen": -2.6812500953674316, + "logits/rejected": -2.721874952316284, + "logps/chosen": -234.8000030517578, + "logps/rejected": -304.20001220703125, + "loss": 0.4785, + "rewards/accuracies": 0.6546212434768677, + "rewards/chosen": -0.39448243379592896, + "rewards/margins": 0.9105468988418579, + "rewards/rejected": -1.3046875, + "step": 2800 + }, + { + "epoch": 0.722193614830072, + "grad_norm": 406.0, + "learning_rate": 1.3890319258496394e-07, + "logits/chosen": -2.7281250953674316, + "logits/rejected": -2.721874952316284, + "logps/chosen": -291.20001220703125, + "logps/rejected": -316.3999938964844, + "loss": 0.6074, + "rewards/accuracies": 0.6696428656578064, + "rewards/chosen": -0.32402342557907104, + "rewards/margins": 0.4481445252895355, + "rewards/rejected": -0.7757812738418579, + "step": 2805 + }, + { + "epoch": 0.723480947476828, + "grad_norm": 708.0, + "learning_rate": 1.3825952626158598e-07, + "logits/chosen": -2.784374952316284, + "logits/rejected": -2.753124952316284, + "logps/chosen": -284.20001220703125, + "logps/rejected": -310.0, + "loss": 0.5699, + "rewards/accuracies": 0.643818736076355, + "rewards/chosen": -0.46992188692092896, + "rewards/margins": 0.831835925579071, + "rewards/rejected": -1.3006470203399658, + "step": 2810 + }, + { + "epoch": 0.724768280123584, + "grad_norm": 448.0, + "learning_rate": 1.3761585993820805e-07, + "logits/chosen": -2.737499952316284, + "logits/rejected": -2.65625, + "logps/chosen": -313.6000061035156, + "logps/rejected": -327.3999938964844, + "loss": 0.5598, + "rewards/accuracies": 0.7114102840423584, + "rewards/chosen": -0.345947265625, + "rewards/margins": 0.7735840082168579, + "rewards/rejected": -1.1208984851837158, + "step": 2815 + }, + { + "epoch": 0.7260556127703398, + "grad_norm": 384.0, + "learning_rate": 1.3697219361483006e-07, + "logits/chosen": -2.25, + "logits/rejected": -2.1578125953674316, + "logps/chosen": -297.6000061035156, + "logps/rejected": -352.6000061035156, + "loss": 0.482, + "rewards/accuracies": 0.7566667199134827, + "rewards/chosen": -0.7378906011581421, + "rewards/margins": 1.201562523841858, + "rewards/rejected": -1.94140625, + "step": 2820 + }, + { + "epoch": 0.7273429454170958, + "grad_norm": 460.0, + "learning_rate": 1.363285272914521e-07, + "logits/chosen": -2.484375, + "logits/rejected": -2.465625047683716, + "logps/chosen": -345.20001220703125, + "logps/rejected": -413.6000061035156, + "loss": 0.4098, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.1375000476837158, + "rewards/margins": 1.3859374523162842, + "rewards/rejected": -2.5218749046325684, + "step": 2825 + }, + { + "epoch": 0.7286302780638517, + "grad_norm": 516.0, + "learning_rate": 1.3568486096807412e-07, + "logits/chosen": -2.5875000953674316, + "logits/rejected": -2.418750047683716, + "logps/chosen": -310.79998779296875, + "logps/rejected": -307.20001220703125, + "loss": 0.4863, + "rewards/accuracies": 0.7563889026641846, + "rewards/chosen": -0.4146484434604645, + "rewards/margins": 1.0526854991912842, + "rewards/rejected": -1.466406226158142, + "step": 2830 + }, + { + "epoch": 0.7299176107106076, + "grad_norm": 504.0, + "learning_rate": 1.350411946446962e-07, + "logits/chosen": -2.6156249046325684, + "logits/rejected": -2.621875047683716, + "logps/chosen": -339.3999938964844, + "logps/rejected": -370.3999938964844, + "loss": 0.4551, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.5677734613418579, + "rewards/margins": 0.9730468988418579, + "rewards/rejected": -1.540624976158142, + "step": 2835 + }, + { + "epoch": 0.7312049433573635, + "grad_norm": 412.0, + "learning_rate": 1.3439752832131823e-07, + "logits/chosen": -2.6156249046325684, + "logits/rejected": -2.653125047683716, + "logps/chosen": -397.20001220703125, + "logps/rejected": -405.20001220703125, + "loss": 0.4352, + "rewards/accuracies": 0.7535713911056519, + "rewards/chosen": -0.4794921875, + "rewards/margins": 1.1101562976837158, + "rewards/rejected": -1.588281273841858, + "step": 2840 + }, + { + "epoch": 0.7324922760041195, + "grad_norm": 924.0, + "learning_rate": 1.3375386199794025e-07, + "logits/chosen": -2.53125, + "logits/rejected": -2.734375, + "logps/chosen": -279.3999938964844, + "logps/rejected": -300.6000061035156, + "loss": 0.5586, + "rewards/accuracies": 0.6101373434066772, + "rewards/chosen": -0.7308593988418579, + "rewards/margins": 0.9410156011581421, + "rewards/rejected": -1.670312523841858, + "step": 2845 + }, + { + "epoch": 0.7337796086508754, + "grad_norm": 468.0, + "learning_rate": 1.331101956745623e-07, + "logits/chosen": -2.6187500953674316, + "logits/rejected": -2.5374999046325684, + "logps/chosen": -275.3999938964844, + "logps/rejected": -320.20001220703125, + "loss": 0.5426, + "rewards/accuracies": 0.6352564096450806, + "rewards/chosen": -0.43159180879592896, + "rewards/margins": 0.811718761920929, + "rewards/rejected": -1.243749976158142, + "step": 2850 + }, + { + "epoch": 0.7350669412976313, + "grad_norm": 464.0, + "learning_rate": 1.3246652935118436e-07, + "logits/chosen": -2.6624999046325684, + "logits/rejected": -2.71875, + "logps/chosen": -316.0, + "logps/rejected": -342.0, + "loss": 0.4002, + "rewards/accuracies": 0.7796429395675659, + "rewards/chosen": -0.655810534954071, + "rewards/margins": 1.1171875, + "rewards/rejected": -1.771875023841858, + "step": 2855 + }, + { + "epoch": 0.7363542739443872, + "grad_norm": 520.0, + "learning_rate": 1.3182286302780638e-07, + "logits/chosen": -2.590625047683716, + "logits/rejected": -2.6156249046325684, + "logps/chosen": -281.1000061035156, + "logps/rejected": -284.25, + "loss": 0.5855, + "rewards/accuracies": 0.6166666746139526, + "rewards/chosen": -0.7484375238418579, + "rewards/margins": 0.684374988079071, + "rewards/rejected": -1.4347655773162842, + "step": 2860 + }, + { + "epoch": 0.7376416065911432, + "grad_norm": 800.0, + "learning_rate": 1.3117919670442842e-07, + "logits/chosen": -2.4593749046325684, + "logits/rejected": -2.5562500953674316, + "logps/chosen": -233.1999969482422, + "logps/rejected": -274.20001220703125, + "loss": 0.534, + "rewards/accuracies": 0.6483333706855774, + "rewards/chosen": -0.4771484434604645, + "rewards/margins": 0.6976562738418579, + "rewards/rejected": -1.1765625476837158, + "step": 2865 + }, + { + "epoch": 0.738928939237899, + "grad_norm": 524.0, + "learning_rate": 1.3053553038105044e-07, + "logits/chosen": -2.621875047683716, + "logits/rejected": -2.596874952316284, + "logps/chosen": -316.79998779296875, + "logps/rejected": -370.20001220703125, + "loss": 0.4629, + "rewards/accuracies": 0.7748918533325195, + "rewards/chosen": -0.5074218511581421, + "rewards/margins": 0.984375, + "rewards/rejected": -1.49365234375, + "step": 2870 + }, + { + "epoch": 0.740216271884655, + "grad_norm": 384.0, + "learning_rate": 1.298918640576725e-07, + "logits/chosen": -2.565624952316284, + "logits/rejected": -2.3843750953674316, + "logps/chosen": -396.79998779296875, + "logps/rejected": -442.0, + "loss": 0.4816, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.86328125, + "rewards/margins": 1.407812476158142, + "rewards/rejected": -2.271484375, + "step": 2875 + }, + { + "epoch": 0.741503604531411, + "grad_norm": 510.0, + "learning_rate": 1.2924819773429455e-07, + "logits/chosen": -2.4375, + "logits/rejected": -2.3296875953674316, + "logps/chosen": -263.20001220703125, + "logps/rejected": -262.20001220703125, + "loss": 0.6109, + "rewards/accuracies": 0.5410714149475098, + "rewards/chosen": -0.6412109136581421, + "rewards/margins": 0.522656261920929, + "rewards/rejected": -1.164453148841858, + "step": 2880 + }, + { + "epoch": 0.7427909371781668, + "grad_norm": 732.0, + "learning_rate": 1.2860453141091657e-07, + "logits/chosen": -2.590625047683716, + "logits/rejected": -2.6312499046325684, + "logps/chosen": -273.6000061035156, + "logps/rejected": -287.20001220703125, + "loss": 0.6961, + "rewards/accuracies": 0.5833333730697632, + "rewards/chosen": -0.5679687261581421, + "rewards/margins": 0.26728516817092896, + "rewards/rejected": -0.836718738079071, + "step": 2885 + }, + { + "epoch": 0.7440782698249228, + "grad_norm": 462.0, + "learning_rate": 1.279608650875386e-07, + "logits/chosen": -2.778125047683716, + "logits/rejected": -2.7562499046325684, + "logps/chosen": -300.0, + "logps/rejected": -348.3999938964844, + "loss": 0.5188, + "rewards/accuracies": 0.6808150410652161, + "rewards/chosen": -0.41289061307907104, + "rewards/margins": 0.819140613079071, + "rewards/rejected": -1.23046875, + "step": 2890 + }, + { + "epoch": 0.7453656024716787, + "grad_norm": 760.0, + "learning_rate": 1.2731719876416068e-07, + "logits/chosen": -2.542187452316284, + "logits/rejected": -2.6343750953674316, + "logps/chosen": -335.20001220703125, + "logps/rejected": -342.6000061035156, + "loss": 0.4799, + "rewards/accuracies": 0.7259052395820618, + "rewards/chosen": -0.33427733182907104, + "rewards/margins": 1.0773437023162842, + "rewards/rejected": -1.411718726158142, + "step": 2895 + }, + { + "epoch": 0.7466529351184346, + "grad_norm": 532.0, + "learning_rate": 1.266735324407827e-07, + "logits/chosen": -2.606250047683716, + "logits/rejected": -2.659374952316284, + "logps/chosen": -330.3999938964844, + "logps/rejected": -460.0, + "loss": 0.4207, + "rewards/accuracies": 0.7734615206718445, + "rewards/chosen": -0.6552734375, + "rewards/margins": 1.1687500476837158, + "rewards/rejected": -1.822656273841858, + "step": 2900 + }, + { + "epoch": 0.7479402677651905, + "grad_norm": 812.0, + "learning_rate": 1.2602986611740474e-07, + "logits/chosen": -2.4281249046325684, + "logits/rejected": -2.418750047683716, + "logps/chosen": -306.0, + "logps/rejected": -321.0, + "loss": 0.4711, + "rewards/accuracies": 0.7517033815383911, + "rewards/chosen": -0.4886474609375, + "rewards/margins": 1.1476562023162842, + "rewards/rejected": -1.635156273841858, + "step": 2905 + }, + { + "epoch": 0.7492276004119465, + "grad_norm": 426.0, + "learning_rate": 1.2538619979402675e-07, + "logits/chosen": -2.5687499046325684, + "logits/rejected": -2.356250047683716, + "logps/chosen": -231.5, + "logps/rejected": -209.5, + "loss": 0.4938, + "rewards/accuracies": 0.7166666984558105, + "rewards/chosen": 0.15273436903953552, + "rewards/margins": 0.7269531488418579, + "rewards/rejected": -0.57568359375, + "step": 2910 + }, + { + "epoch": 0.7505149330587023, + "grad_norm": 576.0, + "learning_rate": 1.247425334706488e-07, + "logits/chosen": -2.640625, + "logits/rejected": -2.512500047683716, + "logps/chosen": -265.0, + "logps/rejected": -328.6000061035156, + "loss": 0.4359, + "rewards/accuracies": 0.7846153974533081, + "rewards/chosen": -0.44316405057907104, + "rewards/margins": 1.0460937023162842, + "rewards/rejected": -1.489843726158142, + "step": 2915 + }, + { + "epoch": 0.7518022657054583, + "grad_norm": 438.0, + "learning_rate": 1.2409886714727084e-07, + "logits/chosen": -2.706249952316284, + "logits/rejected": -2.6968750953674316, + "logps/chosen": -335.79998779296875, + "logps/rejected": -369.6000061035156, + "loss": 0.4566, + "rewards/accuracies": 0.73333340883255, + "rewards/chosen": -0.13642577826976776, + "rewards/margins": 1.15234375, + "rewards/rejected": -1.2882812023162842, + "step": 2920 + }, + { + "epoch": 0.7530895983522142, + "grad_norm": 564.0, + "learning_rate": 1.2345520082389288e-07, + "logits/chosen": -2.625, + "logits/rejected": -2.5999999046325684, + "logps/chosen": -316.6000061035156, + "logps/rejected": -338.79998779296875, + "loss": 0.5074, + "rewards/accuracies": 0.7391667366027832, + "rewards/chosen": -0.4359374940395355, + "rewards/margins": 0.801562488079071, + "rewards/rejected": -1.237158179283142, + "step": 2925 + }, + { + "epoch": 0.7543769309989702, + "grad_norm": 384.0, + "learning_rate": 1.2281153450051492e-07, + "logits/chosen": -2.671875, + "logits/rejected": -2.6156249046325684, + "logps/chosen": -297.3999938964844, + "logps/rejected": -308.0, + "loss": 0.675, + "rewards/accuracies": 0.5618590116500854, + "rewards/chosen": -0.32832032442092896, + "rewards/margins": 0.4994140565395355, + "rewards/rejected": -0.8271484375, + "step": 2930 + }, + { + "epoch": 0.755664263645726, + "grad_norm": 1024.0, + "learning_rate": 1.2216786817713697e-07, + "logits/chosen": -2.7874999046325684, + "logits/rejected": -2.796875, + "logps/chosen": -286.3999938964844, + "logps/rejected": -288.0, + "loss": 0.4279, + "rewards/accuracies": 0.8728896379470825, + "rewards/chosen": -0.45717161893844604, + "rewards/margins": 1.1550781726837158, + "rewards/rejected": -1.6124999523162842, + "step": 2935 + }, + { + "epoch": 0.756951596292482, + "grad_norm": 532.0, + "learning_rate": 1.21524201853759e-07, + "logits/chosen": -2.8125, + "logits/rejected": -2.809375047683716, + "logps/chosen": -350.3999938964844, + "logps/rejected": -343.3999938964844, + "loss": 0.5492, + "rewards/accuracies": 0.6843578219413757, + "rewards/chosen": -0.3561035096645355, + "rewards/margins": 0.7203124761581421, + "rewards/rejected": -1.0773437023162842, + "step": 2940 + }, + { + "epoch": 0.7582389289392379, + "grad_norm": 548.0, + "learning_rate": 1.2088053553038105e-07, + "logits/chosen": -2.690624952316284, + "logits/rejected": -2.559375047683716, + "logps/chosen": -301.0, + "logps/rejected": -311.79998779296875, + "loss": 0.6406, + "rewards/accuracies": 0.6123626232147217, + "rewards/chosen": -0.42973631620407104, + "rewards/margins": 0.46601563692092896, + "rewards/rejected": -0.8960937261581421, + "step": 2945 + }, + { + "epoch": 0.7595262615859938, + "grad_norm": 274.0, + "learning_rate": 1.202368692070031e-07, + "logits/chosen": -2.59375, + "logits/rejected": -2.590625047683716, + "logps/chosen": -297.6000061035156, + "logps/rejected": -355.6000061035156, + "loss": 0.3332, + "rewards/accuracies": 0.8571428060531616, + "rewards/chosen": -0.49162596464157104, + "rewards/margins": 1.420312523841858, + "rewards/rejected": -1.9093749523162842, + "step": 2950 + }, + { + "epoch": 0.7608135942327497, + "grad_norm": 924.0, + "learning_rate": 1.195932028836251e-07, + "logits/chosen": -2.578125, + "logits/rejected": -2.6468749046325684, + "logps/chosen": -305.20001220703125, + "logps/rejected": -324.6000061035156, + "loss": 0.5602, + "rewards/accuracies": 0.6642857789993286, + "rewards/chosen": -0.3824218809604645, + "rewards/margins": 0.758593738079071, + "rewards/rejected": -1.13916015625, + "step": 2955 + }, + { + "epoch": 0.7621009268795057, + "grad_norm": 552.0, + "learning_rate": 1.1894953656024715e-07, + "logits/chosen": -2.606250047683716, + "logits/rejected": -2.5062499046325684, + "logps/chosen": -305.79998779296875, + "logps/rejected": -376.3999938964844, + "loss": 0.5148, + "rewards/accuracies": 0.688076913356781, + "rewards/chosen": -0.82177734375, + "rewards/margins": 0.7578125, + "rewards/rejected": -1.579687476158142, + "step": 2960 + }, + { + "epoch": 0.7633882595262615, + "grad_norm": 456.0, + "learning_rate": 1.1830587023686921e-07, + "logits/chosen": -2.934375047683716, + "logits/rejected": -2.7593750953674316, + "logps/chosen": -288.6000061035156, + "logps/rejected": -265.20001220703125, + "loss": 0.5426, + "rewards/accuracies": 0.6709523797035217, + "rewards/chosen": 0.008410644717514515, + "rewards/margins": 0.696484386920929, + "rewards/rejected": -0.687695324420929, + "step": 2965 + }, + { + "epoch": 0.7646755921730175, + "grad_norm": 398.0, + "learning_rate": 1.1766220391349124e-07, + "logits/chosen": -2.253124952316284, + "logits/rejected": -2.4312500953674316, + "logps/chosen": -328.0, + "logps/rejected": -310.6000061035156, + "loss": 0.4813, + "rewards/accuracies": 0.8300000429153442, + "rewards/chosen": -0.6304687261581421, + "rewards/margins": 1.115234375, + "rewards/rejected": -1.746679663658142, + "step": 2970 + }, + { + "epoch": 0.7659629248197735, + "grad_norm": 520.0, + "learning_rate": 1.1701853759011328e-07, + "logits/chosen": -2.628124952316284, + "logits/rejected": -2.6312499046325684, + "logps/chosen": -341.6000061035156, + "logps/rejected": -367.3999938964844, + "loss": 0.5645, + "rewards/accuracies": 0.7291666865348816, + "rewards/chosen": -0.6439453363418579, + "rewards/margins": 0.8988281488418579, + "rewards/rejected": -1.543359398841858, + "step": 2975 + }, + { + "epoch": 0.7672502574665293, + "grad_norm": 524.0, + "learning_rate": 1.1637487126673531e-07, + "logits/chosen": -2.5374999046325684, + "logits/rejected": -2.5531249046325684, + "logps/chosen": -312.79998779296875, + "logps/rejected": -317.6000061035156, + "loss": 0.4379, + "rewards/accuracies": 0.8369697332382202, + "rewards/chosen": -0.369454950094223, + "rewards/margins": 1.0753905773162842, + "rewards/rejected": -1.4453125, + "step": 2980 + }, + { + "epoch": 0.7685375901132853, + "grad_norm": 816.0, + "learning_rate": 1.1573120494335737e-07, + "logits/chosen": -2.5062499046325684, + "logits/rejected": -2.6468749046325684, + "logps/chosen": -277.79998779296875, + "logps/rejected": -276.6000061035156, + "loss": 0.6129, + "rewards/accuracies": 0.5822222232818604, + "rewards/chosen": -0.42656248807907104, + "rewards/margins": 0.674365222454071, + "rewards/rejected": -1.1003906726837158, + "step": 2985 + }, + { + "epoch": 0.7698249227600412, + "grad_norm": 716.0, + "learning_rate": 1.150875386199794e-07, + "logits/chosen": -2.8187499046325684, + "logits/rejected": -2.7093749046325684, + "logps/chosen": -291.79998779296875, + "logps/rejected": -231.8000030517578, + "loss": 0.552, + "rewards/accuracies": 0.66093909740448, + "rewards/chosen": -0.06689453125, + "rewards/margins": 0.48066407442092896, + "rewards/rejected": -0.5482422113418579, + "step": 2990 + }, + { + "epoch": 0.7711122554067971, + "grad_norm": 556.0, + "learning_rate": 1.1444387229660144e-07, + "logits/chosen": -2.753124952316284, + "logits/rejected": -2.653125047683716, + "logps/chosen": -287.3999938964844, + "logps/rejected": -311.79998779296875, + "loss": 0.5258, + "rewards/accuracies": 0.6757109761238098, + "rewards/chosen": -0.216796875, + "rewards/margins": 0.7730468511581421, + "rewards/rejected": -0.9906250238418579, + "step": 2995 + }, + { + "epoch": 0.772399588053553, + "grad_norm": 824.0, + "learning_rate": 1.1380020597322347e-07, + "logits/chosen": -2.6156249046325684, + "logits/rejected": -2.606250047683716, + "logps/chosen": -351.20001220703125, + "logps/rejected": -355.20001220703125, + "loss": 0.5283, + "rewards/accuracies": 0.6761905550956726, + "rewards/chosen": -0.16879883408546448, + "rewards/margins": 0.718945324420929, + "rewards/rejected": -0.886914074420929, + "step": 3000 + }, + { + "epoch": 0.773686920700309, + "grad_norm": 580.0, + "learning_rate": 1.1315653964984552e-07, + "logits/chosen": -2.6156249046325684, + "logits/rejected": -2.621875047683716, + "logps/chosen": -333.20001220703125, + "logps/rejected": -338.79998779296875, + "loss": 0.577, + "rewards/accuracies": 0.6299999952316284, + "rewards/chosen": -0.43671876192092896, + "rewards/margins": 0.8402343988418579, + "rewards/rejected": -1.275976538658142, + "step": 3005 + }, + { + "epoch": 0.7749742533470649, + "grad_norm": 2032.0, + "learning_rate": 1.1251287332646755e-07, + "logits/chosen": -2.253124952316284, + "logits/rejected": -2.2164063453674316, + "logps/chosen": -298.79998779296875, + "logps/rejected": -342.20001220703125, + "loss": 0.518, + "rewards/accuracies": 0.7061363458633423, + "rewards/chosen": -0.740368664264679, + "rewards/margins": 1.05859375, + "rewards/rejected": -1.799218773841858, + "step": 3010 + }, + { + "epoch": 0.7762615859938208, + "grad_norm": 944.0, + "learning_rate": 1.118692070030896e-07, + "logits/chosen": -2.671875, + "logits/rejected": -2.528125047683716, + "logps/chosen": -308.79998779296875, + "logps/rejected": -341.79998779296875, + "loss": 0.5203, + "rewards/accuracies": 0.6737179756164551, + "rewards/chosen": -0.5314697027206421, + "rewards/margins": 0.8018554449081421, + "rewards/rejected": -1.331298828125, + "step": 3015 + }, + { + "epoch": 0.7775489186405767, + "grad_norm": 362.0, + "learning_rate": 1.1122554067971163e-07, + "logits/chosen": -2.6312499046325684, + "logits/rejected": -2.637500047683716, + "logps/chosen": -316.79998779296875, + "logps/rejected": -300.79998779296875, + "loss": 0.4965, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.6460937261581421, + "rewards/margins": 1.0324218273162842, + "rewards/rejected": -1.6789062023162842, + "step": 3020 + }, + { + "epoch": 0.7788362512873327, + "grad_norm": 804.0, + "learning_rate": 1.1058187435633368e-07, + "logits/chosen": -2.53125, + "logits/rejected": -2.643749952316284, + "logps/chosen": -328.3999938964844, + "logps/rejected": -329.79998779296875, + "loss": 0.4797, + "rewards/accuracies": 0.7649999856948853, + "rewards/chosen": -0.521484375, + "rewards/margins": 0.9921875, + "rewards/rejected": -1.5128905773162842, + "step": 3025 + }, + { + "epoch": 0.7801235839340885, + "grad_norm": 576.0, + "learning_rate": 1.0993820803295571e-07, + "logits/chosen": -2.7093749046325684, + "logits/rejected": -2.549999952316284, + "logps/chosen": -320.20001220703125, + "logps/rejected": -378.0, + "loss": 0.4051, + "rewards/accuracies": 0.8387645483016968, + "rewards/chosen": -0.4867187440395355, + "rewards/margins": 1.3390624523162842, + "rewards/rejected": -1.826562523841858, + "step": 3030 + }, + { + "epoch": 0.7814109165808445, + "grad_norm": 688.0, + "learning_rate": 1.0929454170957775e-07, + "logits/chosen": -2.4906249046325684, + "logits/rejected": -2.5687499046325684, + "logps/chosen": -295.3500061035156, + "logps/rejected": -313.5, + "loss": 0.4764, + "rewards/accuracies": 0.7833333611488342, + "rewards/chosen": -0.6253906488418579, + "rewards/margins": 1.162500023841858, + "rewards/rejected": -1.787500023841858, + "step": 3035 + }, + { + "epoch": 0.7826982492276005, + "grad_norm": 492.0, + "learning_rate": 1.0865087538619978e-07, + "logits/chosen": -2.4625000953674316, + "logits/rejected": -2.515625, + "logps/chosen": -252.39999389648438, + "logps/rejected": -360.0, + "loss": 0.4465, + "rewards/accuracies": 0.759772777557373, + "rewards/chosen": -0.5703125, + "rewards/margins": 1.263281226158142, + "rewards/rejected": -1.8328125476837158, + "step": 3040 + }, + { + "epoch": 0.7839855818743563, + "grad_norm": 438.0, + "learning_rate": 1.0800720906282184e-07, + "logits/chosen": -2.659374952316284, + "logits/rejected": -2.674999952316284, + "logps/chosen": -300.20001220703125, + "logps/rejected": -357.79998779296875, + "loss": 0.5695, + "rewards/accuracies": 0.6483333706855774, + "rewards/chosen": -0.6141601800918579, + "rewards/margins": 0.6947265863418579, + "rewards/rejected": -1.3074219226837158, + "step": 3045 + }, + { + "epoch": 0.7852729145211123, + "grad_norm": 14528.0, + "learning_rate": 1.0736354273944387e-07, + "logits/chosen": -2.706249952316284, + "logits/rejected": -2.721874952316284, + "logps/chosen": -240.60000610351562, + "logps/rejected": -257.0, + "loss": 0.6902, + "rewards/accuracies": 0.6703030467033386, + "rewards/chosen": -0.48066407442092896, + "rewards/margins": 0.47187501192092896, + "rewards/rejected": -0.9507812261581421, + "step": 3050 + }, + { + "epoch": 0.7865602471678682, + "grad_norm": 408.0, + "learning_rate": 1.067198764160659e-07, + "logits/chosen": -2.737499952316284, + "logits/rejected": -2.6171875, + "logps/chosen": -332.0, + "logps/rejected": -391.20001220703125, + "loss": 0.5781, + "rewards/accuracies": 0.6138095855712891, + "rewards/chosen": -0.22988280653953552, + "rewards/margins": 0.7469726800918579, + "rewards/rejected": -0.9769531488418579, + "step": 3055 + }, + { + "epoch": 0.787847579814624, + "grad_norm": 400.0, + "learning_rate": 1.0607621009268794e-07, + "logits/chosen": -2.721874952316284, + "logits/rejected": -2.643749952316284, + "logps/chosen": -323.6000061035156, + "logps/rejected": -372.0, + "loss": 0.4693, + "rewards/accuracies": 0.762225329875946, + "rewards/chosen": -0.369384765625, + "rewards/margins": 1.197656273841858, + "rewards/rejected": -1.568945288658142, + "step": 3060 + }, + { + "epoch": 0.78913491246138, + "grad_norm": 532.0, + "learning_rate": 1.0543254376930998e-07, + "logits/chosen": -2.6187500953674316, + "logits/rejected": -2.512500047683716, + "logps/chosen": -280.0, + "logps/rejected": -336.3999938964844, + "loss": 0.5066, + "rewards/accuracies": 0.7735498547554016, + "rewards/chosen": -0.515917956829071, + "rewards/margins": 1.1281249523162842, + "rewards/rejected": -1.64453125, + "step": 3065 + }, + { + "epoch": 0.790422245108136, + "grad_norm": 520.0, + "learning_rate": 1.0478887744593203e-07, + "logits/chosen": -2.7593750953674316, + "logits/rejected": -2.671875, + "logps/chosen": -295.79998779296875, + "logps/rejected": -246.39999389648438, + "loss": 0.6309, + "rewards/accuracies": 0.7043823003768921, + "rewards/chosen": -0.18154296278953552, + "rewards/margins": 0.4022323489189148, + "rewards/rejected": -0.5843750238418579, + "step": 3070 + }, + { + "epoch": 0.7917095777548918, + "grad_norm": 376.0, + "learning_rate": 1.0414521112255406e-07, + "logits/chosen": -2.659374952316284, + "logits/rejected": -2.5140624046325684, + "logps/chosen": -276.20001220703125, + "logps/rejected": -305.79998779296875, + "loss": 0.4674, + "rewards/accuracies": 0.739242434501648, + "rewards/chosen": -0.37470704317092896, + "rewards/margins": 1.0891602039337158, + "rewards/rejected": -1.4621093273162842, + "step": 3075 + }, + { + "epoch": 0.7929969104016478, + "grad_norm": 616.0, + "learning_rate": 1.035015447991761e-07, + "logits/chosen": -2.6156249046325684, + "logits/rejected": -2.4359374046325684, + "logps/chosen": -247.5, + "logps/rejected": -321.3999938964844, + "loss": 0.5141, + "rewards/accuracies": 0.7692857980728149, + "rewards/chosen": -0.2222011536359787, + "rewards/margins": 0.8150390386581421, + "rewards/rejected": -1.0373046398162842, + "step": 3080 + }, + { + "epoch": 0.7942842430484037, + "grad_norm": 372.0, + "learning_rate": 1.0285787847579814e-07, + "logits/chosen": -2.765625, + "logits/rejected": -2.8218750953674316, + "logps/chosen": -322.3999938964844, + "logps/rejected": -346.20001220703125, + "loss": 0.4691, + "rewards/accuracies": 0.7470238208770752, + "rewards/chosen": -0.3467773497104645, + "rewards/margins": 0.8472656011581421, + "rewards/rejected": -1.192285180091858, + "step": 3085 + }, + { + "epoch": 0.7955715756951597, + "grad_norm": 484.0, + "learning_rate": 1.0221421215242018e-07, + "logits/chosen": -2.59375, + "logits/rejected": -2.5687499046325684, + "logps/chosen": -376.0, + "logps/rejected": -442.0, + "loss": 0.4512, + "rewards/accuracies": 0.7616667151451111, + "rewards/chosen": -1.110937476158142, + "rewards/margins": 1.384374976158142, + "rewards/rejected": -2.4906249046325684, + "step": 3090 + }, + { + "epoch": 0.7968589083419155, + "grad_norm": 636.0, + "learning_rate": 1.0157054582904221e-07, + "logits/chosen": -2.546875, + "logits/rejected": -2.229687452316284, + "logps/chosen": -195.89999389648438, + "logps/rejected": -215.35000610351562, + "loss": 0.6031, + "rewards/accuracies": 0.6767857074737549, + "rewards/chosen": -0.37910157442092896, + "rewards/margins": 0.42695313692092896, + "rewards/rejected": -0.806445300579071, + "step": 3095 + }, + { + "epoch": 0.7981462409886715, + "grad_norm": 474.0, + "learning_rate": 1.0092687950566426e-07, + "logits/chosen": -2.6156249046325684, + "logits/rejected": -2.706249952316284, + "logps/chosen": -269.20001220703125, + "logps/rejected": -319.20001220703125, + "loss": 0.4018, + "rewards/accuracies": 0.8003571629524231, + "rewards/chosen": -0.38818359375, + "rewards/margins": 1.240234375, + "rewards/rejected": -1.631506323814392, + "step": 3100 + }, + { + "epoch": 0.7994335736354274, + "grad_norm": 584.0, + "learning_rate": 1.002832131822863e-07, + "logits/chosen": -2.3890624046325684, + "logits/rejected": -2.2171874046325684, + "logps/chosen": -278.0, + "logps/rejected": -294.3999938964844, + "loss": 0.5262, + "rewards/accuracies": 0.6733333468437195, + "rewards/chosen": -0.3193359375, + "rewards/margins": 0.689453125, + "rewards/rejected": -1.0085937976837158, + "step": 3105 + }, + { + "epoch": 0.8007209062821833, + "grad_norm": 476.0, + "learning_rate": 9.963954685890834e-08, + "logits/chosen": -2.703125, + "logits/rejected": -2.6781249046325684, + "logps/chosen": -305.20001220703125, + "logps/rejected": -362.0, + "loss": 0.4922, + "rewards/accuracies": 0.73807692527771, + "rewards/chosen": -0.5384765863418579, + "rewards/margins": 0.944531261920929, + "rewards/rejected": -1.485937476158142, + "step": 3110 + }, + { + "epoch": 0.8020082389289392, + "grad_norm": 1000.0, + "learning_rate": 9.899588053553037e-08, + "logits/chosen": -2.385937452316284, + "logits/rejected": -2.4390625953674316, + "logps/chosen": -255.1999969482422, + "logps/rejected": -290.6000061035156, + "loss": 0.5004, + "rewards/accuracies": 0.7235714197158813, + "rewards/chosen": -0.5361328125, + "rewards/margins": 0.9161132574081421, + "rewards/rejected": -1.454687476158142, + "step": 3115 + }, + { + "epoch": 0.8032955715756952, + "grad_norm": 588.0, + "learning_rate": 9.835221421215241e-08, + "logits/chosen": -2.481250047683716, + "logits/rejected": -2.401562452316284, + "logps/chosen": -275.6000061035156, + "logps/rejected": -333.20001220703125, + "loss": 0.5496, + "rewards/accuracies": 0.7435897588729858, + "rewards/chosen": -0.755175769329071, + "rewards/margins": 0.80078125, + "rewards/rejected": -1.558203101158142, + "step": 3120 + }, + { + "epoch": 0.804582904222451, + "grad_norm": 480.0, + "learning_rate": 9.770854788877446e-08, + "logits/chosen": -2.518749952316284, + "logits/rejected": -2.3843750953674316, + "logps/chosen": -313.79998779296875, + "logps/rejected": -379.0, + "loss": 0.4652, + "rewards/accuracies": 0.7446428537368774, + "rewards/chosen": -0.762499988079071, + "rewards/margins": 1.112890601158142, + "rewards/rejected": -1.876562476158142, + "step": 3125 + }, + { + "epoch": 0.805870236869207, + "grad_norm": 502.0, + "learning_rate": 9.70648815653965e-08, + "logits/chosen": -2.421875, + "logits/rejected": -2.465625047683716, + "logps/chosen": -356.79998779296875, + "logps/rejected": -383.79998779296875, + "loss": 0.5051, + "rewards/accuracies": 0.7041667103767395, + "rewards/chosen": -0.5755859613418579, + "rewards/margins": 0.955859363079071, + "rewards/rejected": -1.529687523841858, + "step": 3130 + }, + { + "epoch": 0.807157569515963, + "grad_norm": 640.0, + "learning_rate": 9.642121524201853e-08, + "logits/chosen": -2.512500047683716, + "logits/rejected": -2.543750047683716, + "logps/chosen": -348.0, + "logps/rejected": -394.79998779296875, + "loss": 0.4977, + "rewards/accuracies": 0.776309609413147, + "rewards/chosen": -0.8310546875, + "rewards/margins": 0.952343761920929, + "rewards/rejected": -1.779687523841858, + "step": 3135 + }, + { + "epoch": 0.8084449021627188, + "grad_norm": 592.0, + "learning_rate": 9.577754891864057e-08, + "logits/chosen": -2.4124999046325684, + "logits/rejected": -2.4703125953674316, + "logps/chosen": -300.79998779296875, + "logps/rejected": -372.6000061035156, + "loss": 0.4557, + "rewards/accuracies": 0.7022727131843567, + "rewards/chosen": -0.6196533441543579, + "rewards/margins": 1.1121094226837158, + "rewards/rejected": -1.73046875, + "step": 3140 + }, + { + "epoch": 0.8097322348094748, + "grad_norm": 456.0, + "learning_rate": 9.513388259526261e-08, + "logits/chosen": -2.4781250953674316, + "logits/rejected": -2.4749999046325684, + "logps/chosen": -250.89999389648438, + "logps/rejected": -273.6000061035156, + "loss": 0.5512, + "rewards/accuracies": 0.7051648497581482, + "rewards/chosen": -0.5328124761581421, + "rewards/margins": 0.8246093988418579, + "rewards/rejected": -1.3566405773162842, + "step": 3145 + }, + { + "epoch": 0.8110195674562307, + "grad_norm": 732.0, + "learning_rate": 9.449021627188466e-08, + "logits/chosen": -2.489062547683716, + "logits/rejected": -2.409374952316284, + "logps/chosen": -336.20001220703125, + "logps/rejected": -288.79998779296875, + "loss": 0.5641, + "rewards/accuracies": 0.7249783277511597, + "rewards/chosen": -0.5816894769668579, + "rewards/margins": 0.670703113079071, + "rewards/rejected": -1.2531249523162842, + "step": 3150 + }, + { + "epoch": 0.8123069001029866, + "grad_norm": 418.0, + "learning_rate": 9.384654994850669e-08, + "logits/chosen": -2.465625047683716, + "logits/rejected": -2.53125, + "logps/chosen": -266.6000061035156, + "logps/rejected": -309.20001220703125, + "loss": 0.5047, + "rewards/accuracies": 0.7427272796630859, + "rewards/chosen": -0.504101574420929, + "rewards/margins": 1.0390625, + "rewards/rejected": -1.544531226158142, + "step": 3155 + }, + { + "epoch": 0.8135942327497425, + "grad_norm": 354.0, + "learning_rate": 9.320288362512873e-08, + "logits/chosen": -2.8031249046325684, + "logits/rejected": -2.7718749046325684, + "logps/chosen": -275.20001220703125, + "logps/rejected": -386.79998779296875, + "loss": 0.4178, + "rewards/accuracies": 0.7908333539962769, + "rewards/chosen": -0.35661619901657104, + "rewards/margins": 1.2578125, + "rewards/rejected": -1.614843726158142, + "step": 3160 + }, + { + "epoch": 0.8148815653964985, + "grad_norm": 676.0, + "learning_rate": 9.255921730175077e-08, + "logits/chosen": -2.6031250953674316, + "logits/rejected": -2.565624952316284, + "logps/chosen": -306.3999938964844, + "logps/rejected": -345.79998779296875, + "loss": 0.5254, + "rewards/accuracies": 0.6862637400627136, + "rewards/chosen": -0.837109386920929, + "rewards/margins": 0.7964843511581421, + "rewards/rejected": -1.6335937976837158, + "step": 3165 + }, + { + "epoch": 0.8161688980432544, + "grad_norm": 584.0, + "learning_rate": 9.191555097837281e-08, + "logits/chosen": -2.737499952316284, + "logits/rejected": -2.684375047683716, + "logps/chosen": -300.3999938964844, + "logps/rejected": -277.79998779296875, + "loss": 0.5648, + "rewards/accuracies": 0.5680195093154907, + "rewards/chosen": -0.48417967557907104, + "rewards/margins": 0.634960949420929, + "rewards/rejected": -1.1220703125, + "step": 3170 + }, + { + "epoch": 0.8174562306900103, + "grad_norm": 432.0, + "learning_rate": 9.127188465499484e-08, + "logits/chosen": -2.6343750953674316, + "logits/rejected": -2.559375047683716, + "logps/chosen": -335.3999938964844, + "logps/rejected": -295.3999938964844, + "loss": 0.4832, + "rewards/accuracies": 0.784761905670166, + "rewards/chosen": -0.23046875, + "rewards/margins": 0.875, + "rewards/rejected": -1.1062500476837158, + "step": 3175 + }, + { + "epoch": 0.8187435633367662, + "grad_norm": 700.0, + "learning_rate": 9.062821833161689e-08, + "logits/chosen": -2.637500047683716, + "logits/rejected": -2.5843749046325684, + "logps/chosen": -313.79998779296875, + "logps/rejected": -333.6000061035156, + "loss": 0.6344, + "rewards/accuracies": 0.5648077130317688, + "rewards/chosen": -0.4775390625, + "rewards/margins": 0.468719482421875, + "rewards/rejected": -0.9488281011581421, + "step": 3180 + }, + { + "epoch": 0.8200308959835222, + "grad_norm": 684.0, + "learning_rate": 8.998455200823893e-08, + "logits/chosen": -2.5875000953674316, + "logits/rejected": -2.645312547683716, + "logps/chosen": -295.0, + "logps/rejected": -275.20001220703125, + "loss": 0.5359, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.5835937261581421, + "rewards/margins": 0.7914062738418579, + "rewards/rejected": -1.3738281726837158, + "step": 3185 + }, + { + "epoch": 0.821318228630278, + "grad_norm": 474.0, + "learning_rate": 8.934088568486097e-08, + "logits/chosen": -2.7562499046325684, + "logits/rejected": -2.8375000953674316, + "logps/chosen": -313.6000061035156, + "logps/rejected": -301.3999938964844, + "loss": 0.6742, + "rewards/accuracies": 0.6009615659713745, + "rewards/chosen": -0.45136719942092896, + "rewards/margins": 0.3397460877895355, + "rewards/rejected": -0.791015625, + "step": 3190 + }, + { + "epoch": 0.822605561277034, + "grad_norm": 832.0, + "learning_rate": 8.8697219361483e-08, + "logits/chosen": -2.6156249046325684, + "logits/rejected": -2.737499952316284, + "logps/chosen": -312.0, + "logps/rejected": -320.1000061035156, + "loss": 0.5695, + "rewards/accuracies": 0.6609524488449097, + "rewards/chosen": -0.46416014432907104, + "rewards/margins": 0.730273425579071, + "rewards/rejected": -1.195898413658142, + "step": 3195 + }, + { + "epoch": 0.82389289392379, + "grad_norm": 464.0, + "learning_rate": 8.805355303810503e-08, + "logits/chosen": -2.5843749046325684, + "logits/rejected": -2.7093749046325684, + "logps/chosen": -323.3999938964844, + "logps/rejected": -346.8999938964844, + "loss": 0.5285, + "rewards/accuracies": 0.6691666841506958, + "rewards/chosen": -0.27705079317092896, + "rewards/margins": 0.73828125, + "rewards/rejected": -1.0148437023162842, + "step": 3200 + }, + { + "epoch": 0.8251802265705458, + "grad_norm": 536.0, + "learning_rate": 8.740988671472709e-08, + "logits/chosen": -2.575000047683716, + "logits/rejected": -2.53125, + "logps/chosen": -275.6000061035156, + "logps/rejected": -326.3999938964844, + "loss": 0.5879, + "rewards/accuracies": 0.6194047927856445, + "rewards/chosen": -0.6294921636581421, + "rewards/margins": 0.8445800542831421, + "rewards/rejected": -1.4753906726837158, + "step": 3205 + }, + { + "epoch": 0.8264675592173018, + "grad_norm": 524.0, + "learning_rate": 8.676622039134912e-08, + "logits/chosen": -2.8218750953674316, + "logits/rejected": -2.8031249046325684, + "logps/chosen": -385.79998779296875, + "logps/rejected": -388.79998779296875, + "loss": 0.4773, + "rewards/accuracies": 0.7658333778381348, + "rewards/chosen": -0.44511717557907104, + "rewards/margins": 0.919921875, + "rewards/rejected": -1.365625023841858, + "step": 3210 + }, + { + "epoch": 0.8277548918640577, + "grad_norm": 628.0, + "learning_rate": 8.612255406797116e-08, + "logits/chosen": -2.5718750953674316, + "logits/rejected": -2.426562547683716, + "logps/chosen": -325.0, + "logps/rejected": -327.3999938964844, + "loss": 0.6262, + "rewards/accuracies": 0.7546428442001343, + "rewards/chosen": -0.551464855670929, + "rewards/margins": 0.822265625, + "rewards/rejected": -1.3757812976837158, + "step": 3215 + }, + { + "epoch": 0.8290422245108136, + "grad_norm": 600.0, + "learning_rate": 8.547888774459319e-08, + "logits/chosen": -2.534374952316284, + "logits/rejected": -2.450000047683716, + "logps/chosen": -262.3999938964844, + "logps/rejected": -246.8000030517578, + "loss": 0.575, + "rewards/accuracies": 0.6640874147415161, + "rewards/chosen": -0.42451173067092896, + "rewards/margins": 0.48906248807907104, + "rewards/rejected": -0.913281261920929, + "step": 3220 + }, + { + "epoch": 0.8303295571575695, + "grad_norm": 540.0, + "learning_rate": 8.483522142121524e-08, + "logits/chosen": -2.518749952316284, + "logits/rejected": -2.465625047683716, + "logps/chosen": -330.0, + "logps/rejected": -378.3999938964844, + "loss": 0.4641, + "rewards/accuracies": 0.7408334016799927, + "rewards/chosen": -0.7447265386581421, + "rewards/margins": 1.2472655773162842, + "rewards/rejected": -1.9890625476837158, + "step": 3225 + }, + { + "epoch": 0.8316168898043255, + "grad_norm": 450.0, + "learning_rate": 8.419155509783727e-08, + "logits/chosen": -2.6187500953674316, + "logits/rejected": -2.706249952316284, + "logps/chosen": -325.3999938964844, + "logps/rejected": -360.3999938964844, + "loss": 0.4555, + "rewards/accuracies": 0.6555769443511963, + "rewards/chosen": -0.26396483182907104, + "rewards/margins": 1.02734375, + "rewards/rejected": -1.291406273841858, + "step": 3230 + }, + { + "epoch": 0.8329042224510813, + "grad_norm": 408.0, + "learning_rate": 8.354788877445932e-08, + "logits/chosen": -2.621875047683716, + "logits/rejected": -2.4593749046325684, + "logps/chosen": -319.79998779296875, + "logps/rejected": -374.3999938964844, + "loss": 0.4781, + "rewards/accuracies": 0.6855769157409668, + "rewards/chosen": -0.652020275592804, + "rewards/margins": 1.1867187023162842, + "rewards/rejected": -1.83984375, + "step": 3235 + }, + { + "epoch": 0.8341915550978373, + "grad_norm": 320.0, + "learning_rate": 8.290422245108136e-08, + "logits/chosen": -2.7593750953674316, + "logits/rejected": -2.762500047683716, + "logps/chosen": -274.1000061035156, + "logps/rejected": -346.0, + "loss": 0.4504, + "rewards/accuracies": 0.8016666173934937, + "rewards/chosen": -0.34355467557907104, + "rewards/margins": 1.127343773841858, + "rewards/rejected": -1.471093773841858, + "step": 3240 + }, + { + "epoch": 0.8354788877445932, + "grad_norm": 560.0, + "learning_rate": 8.22605561277034e-08, + "logits/chosen": -2.643749952316284, + "logits/rejected": -2.606250047683716, + "logps/chosen": -281.0, + "logps/rejected": -341.6000061035156, + "loss": 0.498, + "rewards/accuracies": 0.7525901794433594, + "rewards/chosen": -0.37421876192092896, + "rewards/margins": 0.925000011920929, + "rewards/rejected": -1.300390601158142, + "step": 3245 + }, + { + "epoch": 0.8367662203913491, + "grad_norm": 484.0, + "learning_rate": 8.161688980432543e-08, + "logits/chosen": -2.5999999046325684, + "logits/rejected": -2.5875000953674316, + "logps/chosen": -332.20001220703125, + "logps/rejected": -342.6000061035156, + "loss": 0.4516, + "rewards/accuracies": 0.7791666984558105, + "rewards/chosen": -0.72412109375, + "rewards/margins": 1.196435570716858, + "rewards/rejected": -1.9191405773162842, + "step": 3250 + }, + { + "epoch": 0.838053553038105, + "grad_norm": 828.0, + "learning_rate": 8.097322348094747e-08, + "logits/chosen": -2.793750047683716, + "logits/rejected": -2.1527342796325684, + "logps/chosen": -253.625, + "logps/rejected": -271.20001220703125, + "loss": 0.5258, + "rewards/accuracies": 0.7636722326278687, + "rewards/chosen": -0.4361328184604645, + "rewards/margins": 0.712109386920929, + "rewards/rejected": -1.1501953601837158, + "step": 3255 + }, + { + "epoch": 0.839340885684861, + "grad_norm": 324.0, + "learning_rate": 8.032955715756952e-08, + "logits/chosen": -2.700000047683716, + "logits/rejected": -2.778125047683716, + "logps/chosen": -323.20001220703125, + "logps/rejected": -404.6000061035156, + "loss": 0.5363, + "rewards/accuracies": 0.6270604729652405, + "rewards/chosen": -0.5433593988418579, + "rewards/margins": 0.8687499761581421, + "rewards/rejected": -1.4132812023162842, + "step": 3260 + }, + { + "epoch": 0.8406282183316169, + "grad_norm": 720.0, + "learning_rate": 7.968589083419156e-08, + "logits/chosen": -2.659374952316284, + "logits/rejected": -2.606250047683716, + "logps/chosen": -345.6000061035156, + "logps/rejected": -383.20001220703125, + "loss": 0.482, + "rewards/accuracies": 0.7583333253860474, + "rewards/chosen": -0.9585937261581421, + "rewards/margins": 1.1980469226837158, + "rewards/rejected": -2.153125047683716, + "step": 3265 + }, + { + "epoch": 0.8419155509783728, + "grad_norm": 422.0, + "learning_rate": 7.904222451081359e-08, + "logits/chosen": -2.481250047683716, + "logits/rejected": -2.6656250953674316, + "logps/chosen": -341.6000061035156, + "logps/rejected": -330.0, + "loss": 0.5906, + "rewards/accuracies": 0.6751224398612976, + "rewards/chosen": -0.4034179747104645, + "rewards/margins": 0.749218761920929, + "rewards/rejected": -1.151953101158142, + "step": 3270 + }, + { + "epoch": 0.8432028836251287, + "grad_norm": 332.0, + "learning_rate": 7.839855818743563e-08, + "logits/chosen": -2.59375, + "logits/rejected": -2.6328125, + "logps/chosen": -295.79998779296875, + "logps/rejected": -303.3999938964844, + "loss": 0.4383, + "rewards/accuracies": 0.7499450445175171, + "rewards/chosen": -0.24394531548023224, + "rewards/margins": 1.014062523841858, + "rewards/rejected": -1.257226586341858, + "step": 3275 + }, + { + "epoch": 0.8444902162718847, + "grad_norm": 712.0, + "learning_rate": 7.775489186405767e-08, + "logits/chosen": -2.440624952316284, + "logits/rejected": -2.46875, + "logps/chosen": -308.5, + "logps/rejected": -303.6000061035156, + "loss": 0.573, + "rewards/accuracies": 0.6699999570846558, + "rewards/chosen": -0.47919923067092896, + "rewards/margins": 0.744140625, + "rewards/rejected": -1.2218749523162842, + "step": 3280 + }, + { + "epoch": 0.8457775489186405, + "grad_norm": 376.0, + "learning_rate": 7.711122554067972e-08, + "logits/chosen": -2.640625, + "logits/rejected": -2.5843749046325684, + "logps/chosen": -300.6000061035156, + "logps/rejected": -337.20001220703125, + "loss": 0.6289, + "rewards/accuracies": 0.6191667318344116, + "rewards/chosen": -0.7855468988418579, + "rewards/margins": 0.494140625, + "rewards/rejected": -1.2800781726837158, + "step": 3285 + }, + { + "epoch": 0.8470648815653965, + "grad_norm": 430.0, + "learning_rate": 7.646755921730175e-08, + "logits/chosen": -2.6624999046325684, + "logits/rejected": -2.6500000953674316, + "logps/chosen": -340.0, + "logps/rejected": -401.20001220703125, + "loss": 0.5625, + "rewards/accuracies": 0.6769047975540161, + "rewards/chosen": -0.49604493379592896, + "rewards/margins": 0.735156238079071, + "rewards/rejected": -1.2294921875, + "step": 3290 + }, + { + "epoch": 0.8483522142121525, + "grad_norm": 432.0, + "learning_rate": 7.582389289392379e-08, + "logits/chosen": -2.6468749046325684, + "logits/rejected": -2.690624952316284, + "logps/chosen": -357.6000061035156, + "logps/rejected": -459.20001220703125, + "loss": 0.4377, + "rewards/accuracies": 0.8183334469795227, + "rewards/chosen": -0.616015613079071, + "rewards/margins": 1.4921875, + "rewards/rejected": -2.1117186546325684, + "step": 3295 + }, + { + "epoch": 0.8496395468589083, + "grad_norm": 692.0, + "learning_rate": 7.518022657054583e-08, + "logits/chosen": -2.512500047683716, + "logits/rejected": -2.432812452316284, + "logps/chosen": -301.6000061035156, + "logps/rejected": -321.0, + "loss": 0.4898, + "rewards/accuracies": 0.7791666984558105, + "rewards/chosen": -0.33720701932907104, + "rewards/margins": 0.764843761920929, + "rewards/rejected": -1.0988280773162842, + "step": 3300 + }, + { + "epoch": 0.8509268795056643, + "grad_norm": 724.0, + "learning_rate": 7.453656024716787e-08, + "logits/chosen": -2.9156250953674316, + "logits/rejected": -2.871875047683716, + "logps/chosen": -329.20001220703125, + "logps/rejected": -365.6000061035156, + "loss": 0.5793, + "rewards/accuracies": 0.6659615635871887, + "rewards/chosen": -0.142822265625, + "rewards/margins": 0.655468761920929, + "rewards/rejected": -0.797656238079071, + "step": 3305 + }, + { + "epoch": 0.8522142121524202, + "grad_norm": 488.0, + "learning_rate": 7.38928939237899e-08, + "logits/chosen": -2.762500047683716, + "logits/rejected": -2.7906250953674316, + "logps/chosen": -376.0, + "logps/rejected": -342.79998779296875, + "loss": 0.5348, + "rewards/accuracies": 0.6396428346633911, + "rewards/chosen": -0.35624998807907104, + "rewards/margins": 0.669726550579071, + "rewards/rejected": -1.02734375, + "step": 3310 + }, + { + "epoch": 0.8535015447991761, + "grad_norm": 444.0, + "learning_rate": 7.324922760041195e-08, + "logits/chosen": -2.721874952316284, + "logits/rejected": -2.78125, + "logps/chosen": -318.20001220703125, + "logps/rejected": -380.79998779296875, + "loss": 0.4961, + "rewards/accuracies": 0.7439935803413391, + "rewards/chosen": -0.29863280057907104, + "rewards/margins": 0.881640613079071, + "rewards/rejected": -1.1796875, + "step": 3315 + }, + { + "epoch": 0.854788877445932, + "grad_norm": 564.0, + "learning_rate": 7.260556127703399e-08, + "logits/chosen": -2.4749999046325684, + "logits/rejected": -2.4749999046325684, + "logps/chosen": -308.79998779296875, + "logps/rejected": -383.6000061035156, + "loss": 0.3646, + "rewards/accuracies": 0.8783333897590637, + "rewards/chosen": -0.869921863079071, + "rewards/margins": 1.55078125, + "rewards/rejected": -2.418750047683716, + "step": 3320 + }, + { + "epoch": 0.856076210092688, + "grad_norm": 253.0, + "learning_rate": 7.196189495365603e-08, + "logits/chosen": -2.3968749046325684, + "logits/rejected": -2.4000000953674316, + "logps/chosen": -325.20001220703125, + "logps/rejected": -446.79998779296875, + "loss": 0.3426, + "rewards/accuracies": 0.8359615206718445, + "rewards/chosen": -0.994140625, + "rewards/margins": 1.7570312023162842, + "rewards/rejected": -2.753124952316284, + "step": 3325 + }, + { + "epoch": 0.8573635427394438, + "grad_norm": 624.0, + "learning_rate": 7.131822863027806e-08, + "logits/chosen": -2.5250000953674316, + "logits/rejected": -2.5093750953674316, + "logps/chosen": -365.6000061035156, + "logps/rejected": -417.6000061035156, + "loss": 0.4609, + "rewards/accuracies": 0.7616666555404663, + "rewards/chosen": -0.9554687738418579, + "rewards/margins": 1.267968773841858, + "rewards/rejected": -2.223437547683716, + "step": 3330 + }, + { + "epoch": 0.8586508753861998, + "grad_norm": 832.0, + "learning_rate": 7.067456230690009e-08, + "logits/chosen": -2.53125, + "logits/rejected": -2.59375, + "logps/chosen": -254.89999389648438, + "logps/rejected": -292.3999938964844, + "loss": 0.575, + "rewards/accuracies": 0.7133333683013916, + "rewards/chosen": -0.6749511957168579, + "rewards/margins": 0.75, + "rewards/rejected": -1.4249999523162842, + "step": 3335 + }, + { + "epoch": 0.8599382080329557, + "grad_norm": 372.0, + "learning_rate": 7.003089598352215e-08, + "logits/chosen": -2.5687499046325684, + "logits/rejected": -2.721874952316284, + "logps/chosen": -297.79998779296875, + "logps/rejected": -300.0, + "loss": 0.5617, + "rewards/accuracies": 0.69989013671875, + "rewards/chosen": -0.3876953125, + "rewards/margins": 0.65625, + "rewards/rejected": -1.044531226158142, + "step": 3340 + }, + { + "epoch": 0.8612255406797117, + "grad_norm": 506.0, + "learning_rate": 6.938722966014417e-08, + "logits/chosen": -2.5625, + "logits/rejected": -2.690624952316284, + "logps/chosen": -313.6000061035156, + "logps/rejected": -329.6000061035156, + "loss": 0.4422, + "rewards/accuracies": 0.7230769395828247, + "rewards/chosen": -0.624804675579071, + "rewards/margins": 1.067968726158142, + "rewards/rejected": -1.69140625, + "step": 3345 + }, + { + "epoch": 0.8625128733264675, + "grad_norm": 532.0, + "learning_rate": 6.874356333676622e-08, + "logits/chosen": -2.731250047683716, + "logits/rejected": -2.5999999046325684, + "logps/chosen": -354.0, + "logps/rejected": -353.20001220703125, + "loss": 0.582, + "rewards/accuracies": 0.65666663646698, + "rewards/chosen": -0.3546386659145355, + "rewards/margins": 0.6539062261581421, + "rewards/rejected": -1.010156273841858, + "step": 3350 + }, + { + "epoch": 0.8638002059732235, + "grad_norm": 524.0, + "learning_rate": 6.809989701338825e-08, + "logits/chosen": -2.528125047683716, + "logits/rejected": -2.526562452316284, + "logps/chosen": -342.3999938964844, + "logps/rejected": -410.3999938964844, + "loss": 0.4285, + "rewards/accuracies": 0.7950000166893005, + "rewards/chosen": -0.5337890386581421, + "rewards/margins": 1.252343773841858, + "rewards/rejected": -1.7843749523162842, + "step": 3355 + }, + { + "epoch": 0.8650875386199794, + "grad_norm": 604.0, + "learning_rate": 6.74562306900103e-08, + "logits/chosen": NaN, + "logits/rejected": -2.26171875, + "logps/chosen": -293.79998779296875, + "logps/rejected": -312.8999938964844, + "loss": 0.5137, + "rewards/accuracies": 0.7217424511909485, + "rewards/chosen": -0.4085937440395355, + "rewards/margins": 0.91015625, + "rewards/rejected": -1.318750023841858, + "step": 3360 + }, + { + "epoch": 0.8663748712667353, + "grad_norm": 334.0, + "learning_rate": 6.681256436663233e-08, + "logits/chosen": -2.590625047683716, + "logits/rejected": -2.424999952316284, + "logps/chosen": -314.20001220703125, + "logps/rejected": -359.3999938964844, + "loss": 0.3428, + "rewards/accuracies": 0.8984848856925964, + "rewards/chosen": -0.41972655057907104, + "rewards/margins": 1.528906226158142, + "rewards/rejected": -1.9500000476837158, + "step": 3365 + }, + { + "epoch": 0.8676622039134912, + "grad_norm": 592.0, + "learning_rate": 6.616889804325438e-08, + "logits/chosen": -2.6968750953674316, + "logits/rejected": -2.746875047683716, + "logps/chosen": -322.1000061035156, + "logps/rejected": -262.1000061035156, + "loss": 0.5352, + "rewards/accuracies": 0.6956044435501099, + "rewards/chosen": -0.18476562201976776, + "rewards/margins": 0.565234363079071, + "rewards/rejected": -0.748828113079071, + "step": 3370 + }, + { + "epoch": 0.8689495365602472, + "grad_norm": 430.0, + "learning_rate": 6.55252317198764e-08, + "logits/chosen": -2.684375047683716, + "logits/rejected": -2.456249952316284, + "logps/chosen": -303.3999938964844, + "logps/rejected": -320.0, + "loss": 0.5199, + "rewards/accuracies": 0.7129120826721191, + "rewards/chosen": -0.4246582090854645, + "rewards/margins": 0.769726574420929, + "rewards/rejected": -1.193750023841858, + "step": 3375 + }, + { + "epoch": 0.870236869207003, + "grad_norm": 438.0, + "learning_rate": 6.488156539649846e-08, + "logits/chosen": -2.703125, + "logits/rejected": -2.762500047683716, + "logps/chosen": -323.3999938964844, + "logps/rejected": -290.6000061035156, + "loss": 0.593, + "rewards/accuracies": 0.6830952763557434, + "rewards/chosen": -0.4139038026332855, + "rewards/margins": 0.44536131620407104, + "rewards/rejected": -0.8589843511581421, + "step": 3380 + }, + { + "epoch": 0.871524201853759, + "grad_norm": 392.0, + "learning_rate": 6.423789907312049e-08, + "logits/chosen": -2.590625047683716, + "logits/rejected": -2.453125, + "logps/chosen": -310.6000061035156, + "logps/rejected": -366.79998779296875, + "loss": 0.4637, + "rewards/accuracies": 0.7840034365653992, + "rewards/chosen": -0.6812499761581421, + "rewards/margins": 1.1359374523162842, + "rewards/rejected": -1.8171875476837158, + "step": 3385 + }, + { + "epoch": 0.872811534500515, + "grad_norm": 724.0, + "learning_rate": 6.359423274974253e-08, + "logits/chosen": -2.2874999046325684, + "logits/rejected": -2.4937500953674316, + "logps/chosen": -309.3999938964844, + "logps/rejected": -353.79998779296875, + "loss": 0.427, + "rewards/accuracies": 0.8417307734489441, + "rewards/chosen": -0.7630859613418579, + "rewards/margins": 1.177343726158142, + "rewards/rejected": -1.9425780773162842, + "step": 3390 + }, + { + "epoch": 0.8740988671472708, + "grad_norm": 450.0, + "learning_rate": 6.295056642636456e-08, + "logits/chosen": -2.6624999046325684, + "logits/rejected": -2.6796875, + "logps/chosen": -387.20001220703125, + "logps/rejected": -361.20001220703125, + "loss": 0.5625, + "rewards/accuracies": 0.6966208815574646, + "rewards/chosen": -0.21132811903953552, + "rewards/margins": 0.5843750238418579, + "rewards/rejected": -0.7963622808456421, + "step": 3395 + }, + { + "epoch": 0.8753861997940268, + "grad_norm": 696.0, + "learning_rate": 6.23069001029866e-08, + "logits/chosen": -2.512500047683716, + "logits/rejected": -1.9660155773162842, + "logps/chosen": -285.79998779296875, + "logps/rejected": -326.79998779296875, + "loss": 0.4797, + "rewards/accuracies": 0.7083333730697632, + "rewards/chosen": -0.5462890863418579, + "rewards/margins": 1.072656273841858, + "rewards/rejected": -1.6160156726837158, + "step": 3400 + }, + { + "epoch": 0.8766735324407827, + "grad_norm": 456.0, + "learning_rate": 6.166323377960865e-08, + "logits/chosen": -2.5250000953674316, + "logits/rejected": -2.59375, + "logps/chosen": -364.79998779296875, + "logps/rejected": -376.79998779296875, + "loss": 0.5664, + "rewards/accuracies": 0.6391667127609253, + "rewards/chosen": -0.500292956829071, + "rewards/margins": 0.6685546636581421, + "rewards/rejected": -1.1687500476837158, + "step": 3405 + }, + { + "epoch": 0.8779608650875386, + "grad_norm": 404.0, + "learning_rate": 6.101956745623069e-08, + "logits/chosen": -2.6312499046325684, + "logits/rejected": -2.6937499046325684, + "logps/chosen": -337.3999938964844, + "logps/rejected": -364.20001220703125, + "loss": 0.4895, + "rewards/accuracies": 0.7908333539962769, + "rewards/chosen": -0.734570324420929, + "rewards/margins": 1.2195312976837158, + "rewards/rejected": -1.94921875, + "step": 3410 + }, + { + "epoch": 0.8792481977342945, + "grad_norm": 446.0, + "learning_rate": 6.037590113285273e-08, + "logits/chosen": -2.6968750953674316, + "logits/rejected": -2.387500047683716, + "logps/chosen": -237.60000610351562, + "logps/rejected": -279.79998779296875, + "loss": 0.5687, + "rewards/accuracies": 0.6832234263420105, + "rewards/chosen": -0.375, + "rewards/margins": 0.639843761920929, + "rewards/rejected": -1.0144531726837158, + "step": 3415 + }, + { + "epoch": 0.8805355303810505, + "grad_norm": 912.0, + "learning_rate": 5.973223480947476e-08, + "logits/chosen": -2.4296875, + "logits/rejected": -2.125, + "logps/chosen": -253.39999389648438, + "logps/rejected": -263.6000061035156, + "loss": 0.5301, + "rewards/accuracies": 0.6941239237785339, + "rewards/chosen": 0.04282226413488388, + "rewards/margins": 0.558300793170929, + "rewards/rejected": -0.515063464641571, + "step": 3420 + }, + { + "epoch": 0.8818228630278064, + "grad_norm": 350.0, + "learning_rate": 5.9088568486096805e-08, + "logits/chosen": -2.609375, + "logits/rejected": -2.495312452316284, + "logps/chosen": -355.6000061035156, + "logps/rejected": -312.3999938964844, + "loss": 0.5629, + "rewards/accuracies": 0.7089394330978394, + "rewards/chosen": -0.52099609375, + "rewards/margins": 0.5887695550918579, + "rewards/rejected": -1.107812523841858, + "step": 3425 + }, + { + "epoch": 0.8831101956745623, + "grad_norm": 444.0, + "learning_rate": 5.844490216271884e-08, + "logits/chosen": -2.4937500953674316, + "logits/rejected": -2.390625, + "logps/chosen": -353.20001220703125, + "logps/rejected": -407.6000061035156, + "loss": 0.4121, + "rewards/accuracies": 0.8177198171615601, + "rewards/chosen": -0.5921875238418579, + "rewards/margins": 1.314843773841858, + "rewards/rejected": -1.9050781726837158, + "step": 3430 + }, + { + "epoch": 0.8843975283213182, + "grad_norm": 302.0, + "learning_rate": 5.7801235839340884e-08, + "logits/chosen": -2.2906250953674316, + "logits/rejected": -2.534374952316284, + "logps/chosen": -219.8000030517578, + "logps/rejected": -204.35000610351562, + "loss": 0.6, + "rewards/accuracies": 0.511904776096344, + "rewards/chosen": 0.06660155951976776, + "rewards/margins": 0.3775390684604645, + "rewards/rejected": -0.31083983182907104, + "step": 3435 + }, + { + "epoch": 0.8856848609680742, + "grad_norm": 368.0, + "learning_rate": 5.715756951596292e-08, + "logits/chosen": -2.565624952316284, + "logits/rejected": -2.684375047683716, + "logps/chosen": -263.8999938964844, + "logps/rejected": -332.70001220703125, + "loss": 0.4506, + "rewards/accuracies": 0.7900000810623169, + "rewards/chosen": -0.21250000596046448, + "rewards/margins": 1.058203101158142, + "rewards/rejected": -1.270440697669983, + "step": 3440 + }, + { + "epoch": 0.88697219361483, + "grad_norm": 544.0, + "learning_rate": 5.651390319258496e-08, + "logits/chosen": -2.4156250953674316, + "logits/rejected": -2.6312499046325684, + "logps/chosen": -264.6000061035156, + "logps/rejected": -371.0, + "loss": 0.4211, + "rewards/accuracies": 0.8392857313156128, + "rewards/chosen": -0.3070312440395355, + "rewards/margins": 1.2421875, + "rewards/rejected": -1.5480468273162842, + "step": 3445 + }, + { + "epoch": 0.888259526261586, + "grad_norm": 784.0, + "learning_rate": 5.5870236869207e-08, + "logits/chosen": -2.578125, + "logits/rejected": -2.7562499046325684, + "logps/chosen": -279.0, + "logps/rejected": -237.39999389648438, + "loss": 0.4773, + "rewards/accuracies": 0.7569698095321655, + "rewards/chosen": -0.19003906846046448, + "rewards/margins": 0.8753906488418579, + "rewards/rejected": -1.064843773841858, + "step": 3450 + }, + { + "epoch": 0.889546858908342, + "grad_norm": 664.0, + "learning_rate": 5.522657054582904e-08, + "logits/chosen": -2.53125, + "logits/rejected": -2.5843749046325684, + "logps/chosen": -299.6000061035156, + "logps/rejected": -353.6000061035156, + "loss": 0.5215, + "rewards/accuracies": 0.7314286231994629, + "rewards/chosen": -0.6332031488418579, + "rewards/margins": 0.96826171875, + "rewards/rejected": -1.605078101158142, + "step": 3455 + }, + { + "epoch": 0.8908341915550978, + "grad_norm": 556.0, + "learning_rate": 5.458290422245108e-08, + "logits/chosen": -2.6187500953674316, + "logits/rejected": -2.640625, + "logps/chosen": -362.3999938964844, + "logps/rejected": -357.6000061035156, + "loss": 0.5586, + "rewards/accuracies": 0.7035714387893677, + "rewards/chosen": -0.782031238079071, + "rewards/margins": 0.685546875, + "rewards/rejected": -1.46875, + "step": 3460 + }, + { + "epoch": 0.8921215242018538, + "grad_norm": 472.0, + "learning_rate": 5.393923789907312e-08, + "logits/chosen": -2.746875047683716, + "logits/rejected": -2.612499952316284, + "logps/chosen": -230.8000030517578, + "logps/rejected": -299.6000061035156, + "loss": 0.527, + "rewards/accuracies": 0.6965079307556152, + "rewards/chosen": -0.37771910429000854, + "rewards/margins": 0.865234375, + "rewards/rejected": -1.2453124523162842, + "step": 3465 + }, + { + "epoch": 0.8934088568486097, + "grad_norm": 396.0, + "learning_rate": 5.3295571575695156e-08, + "logits/chosen": -2.503124952316284, + "logits/rejected": -2.4234375953674316, + "logps/chosen": -273.79998779296875, + "logps/rejected": -341.79998779296875, + "loss": 0.4898, + "rewards/accuracies": 0.8174999952316284, + "rewards/chosen": -0.619189441204071, + "rewards/margins": 1.2160155773162842, + "rewards/rejected": -1.836328148841858, + "step": 3470 + }, + { + "epoch": 0.8946961894953656, + "grad_norm": 480.0, + "learning_rate": 5.26519052523172e-08, + "logits/chosen": -2.609375, + "logits/rejected": -2.5250000953674316, + "logps/chosen": -336.3999938964844, + "logps/rejected": -362.3999938964844, + "loss": 0.5109, + "rewards/accuracies": 0.6981044411659241, + "rewards/chosen": -0.26640623807907104, + "rewards/margins": 0.783984363079071, + "rewards/rejected": -1.050390601158142, + "step": 3475 + }, + { + "epoch": 0.8959835221421215, + "grad_norm": 640.0, + "learning_rate": 5.2008238928939235e-08, + "logits/chosen": -2.653125047683716, + "logits/rejected": -2.5062499046325684, + "logps/chosen": -323.6000061035156, + "logps/rejected": -353.6000061035156, + "loss": 0.5418, + "rewards/accuracies": 0.6349999904632568, + "rewards/chosen": -0.45263671875, + "rewards/margins": 0.7632812261581421, + "rewards/rejected": -1.2160155773162842, + "step": 3480 + }, + { + "epoch": 0.8972708547888775, + "grad_norm": 468.0, + "learning_rate": 5.136457260556128e-08, + "logits/chosen": -2.421875, + "logits/rejected": -2.5234375, + "logps/chosen": -282.3999938964844, + "logps/rejected": -329.20001220703125, + "loss": 0.4508, + "rewards/accuracies": 0.7128571271896362, + "rewards/chosen": -0.3004394471645355, + "rewards/margins": 1.1843750476837158, + "rewards/rejected": -1.485937476158142, + "step": 3485 + }, + { + "epoch": 0.8985581874356333, + "grad_norm": 568.0, + "learning_rate": 5.0720906282183313e-08, + "logits/chosen": -2.3125, + "logits/rejected": -2.4000000953674316, + "logps/chosen": -325.20001220703125, + "logps/rejected": -407.6000061035156, + "loss": 0.4465, + "rewards/accuracies": 0.7398015856742859, + "rewards/chosen": -0.981640636920929, + "rewards/margins": 1.3976562023162842, + "rewards/rejected": -2.379687547683716, + "step": 3490 + }, + { + "epoch": 0.8998455200823893, + "grad_norm": 424.0, + "learning_rate": 5.0077239958805356e-08, + "logits/chosen": -2.8187499046325684, + "logits/rejected": -2.825000047683716, + "logps/chosen": -364.79998779296875, + "logps/rejected": -336.0, + "loss": 0.5051, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.100189208984375, + "rewards/margins": 0.9476562738418579, + "rewards/rejected": -1.0499999523162842, + "step": 3495 + }, + { + "epoch": 0.9011328527291452, + "grad_norm": 592.0, + "learning_rate": 4.943357363542739e-08, + "logits/chosen": -2.5374999046325684, + "logits/rejected": -2.421875, + "logps/chosen": -312.3999938964844, + "logps/rejected": -339.6000061035156, + "loss": 0.5887, + "rewards/accuracies": 0.6460714936256409, + "rewards/chosen": -0.4244140684604645, + "rewards/margins": 0.915234386920929, + "rewards/rejected": -1.3390624523162842, + "step": 3500 + }, + { + "epoch": 0.9024201853759012, + "grad_norm": 462.0, + "learning_rate": 4.8789907312049435e-08, + "logits/chosen": -2.731250047683716, + "logits/rejected": -2.668750047683716, + "logps/chosen": -352.79998779296875, + "logps/rejected": -397.6000061035156, + "loss": 0.4375, + "rewards/accuracies": 0.7467033267021179, + "rewards/chosen": -0.39787596464157104, + "rewards/margins": 1.185937523841858, + "rewards/rejected": -1.5851562023162842, + "step": 3505 + }, + { + "epoch": 0.903707518022657, + "grad_norm": 482.0, + "learning_rate": 4.814624098867147e-08, + "logits/chosen": -2.534374952316284, + "logits/rejected": -2.7406249046325684, + "logps/chosen": -285.29998779296875, + "logps/rejected": -317.79998779296875, + "loss": 0.5734, + "rewards/accuracies": 0.5839744210243225, + "rewards/chosen": -0.44140625, + "rewards/margins": 0.8138672113418579, + "rewards/rejected": -1.2531249523162842, + "step": 3510 + }, + { + "epoch": 0.904994850669413, + "grad_norm": 556.0, + "learning_rate": 4.7502574665293514e-08, + "logits/chosen": -2.609375, + "logits/rejected": -2.7093749046325684, + "logps/chosen": -368.79998779296875, + "logps/rejected": -413.20001220703125, + "loss": 0.5344, + "rewards/accuracies": 0.7101190686225891, + "rewards/chosen": -0.8929687738418579, + "rewards/margins": 1.127343773841858, + "rewards/rejected": -2.0234375, + "step": 3515 + }, + { + "epoch": 0.9062821833161689, + "grad_norm": 664.0, + "learning_rate": 4.685890834191555e-08, + "logits/chosen": -2.5062499046325684, + "logits/rejected": -2.426562547683716, + "logps/chosen": -307.0, + "logps/rejected": -408.0, + "loss": 0.3256, + "rewards/accuracies": 0.8225000500679016, + "rewards/chosen": -0.71337890625, + "rewards/margins": 1.7570312023162842, + "rewards/rejected": -2.47265625, + "step": 3520 + }, + { + "epoch": 0.9075695159629248, + "grad_norm": 1040.0, + "learning_rate": 4.621524201853759e-08, + "logits/chosen": -2.4625000953674316, + "logits/rejected": -2.659374952316284, + "logps/chosen": -338.79998779296875, + "logps/rejected": -400.0, + "loss": 0.5408, + "rewards/accuracies": 0.6821428537368774, + "rewards/chosen": -0.842968761920929, + "rewards/margins": 0.8666015863418579, + "rewards/rejected": -1.708593726158142, + "step": 3525 + }, + { + "epoch": 0.9088568486096807, + "grad_norm": 440.0, + "learning_rate": 4.557157569515963e-08, + "logits/chosen": -2.671875, + "logits/rejected": -2.703125, + "logps/chosen": -339.20001220703125, + "logps/rejected": -375.3999938964844, + "loss": 0.4918, + "rewards/accuracies": 0.7684615850448608, + "rewards/chosen": -0.5479491949081421, + "rewards/margins": 0.9794921875, + "rewards/rejected": -1.52734375, + "step": 3530 + }, + { + "epoch": 0.9101441812564367, + "grad_norm": 656.0, + "learning_rate": 4.492790937178167e-08, + "logits/chosen": -2.5, + "logits/rejected": -2.4937500953674316, + "logps/chosen": -281.3999938964844, + "logps/rejected": -368.79998779296875, + "loss": 0.448, + "rewards/accuracies": 0.7783333659172058, + "rewards/chosen": -0.971386730670929, + "rewards/margins": 1.27734375, + "rewards/rejected": -2.2476563453674316, + "step": 3535 + }, + { + "epoch": 0.9114315139031925, + "grad_norm": 288.0, + "learning_rate": 4.42842430484037e-08, + "logits/chosen": -2.53125, + "logits/rejected": -2.5406250953674316, + "logps/chosen": -308.3999938964844, + "logps/rejected": -336.79998779296875, + "loss": 0.5055, + "rewards/accuracies": 0.7377381324768066, + "rewards/chosen": -0.3960937559604645, + "rewards/margins": 0.8785156011581421, + "rewards/rejected": -1.2734375, + "step": 3540 + }, + { + "epoch": 0.9127188465499485, + "grad_norm": 760.0, + "learning_rate": 4.364057672502574e-08, + "logits/chosen": -2.6031250953674316, + "logits/rejected": -2.721874952316284, + "logps/chosen": -373.6000061035156, + "logps/rejected": -420.79998779296875, + "loss": 0.5023, + "rewards/accuracies": 0.7360714673995972, + "rewards/chosen": -0.552734375, + "rewards/margins": 0.9476562738418579, + "rewards/rejected": -1.501562476158142, + "step": 3545 + }, + { + "epoch": 0.9140061791967045, + "grad_norm": 452.0, + "learning_rate": 4.299691040164778e-08, + "logits/chosen": -2.6875, + "logits/rejected": -2.706249952316284, + "logps/chosen": -342.3999938964844, + "logps/rejected": -384.0, + "loss": 0.5941, + "rewards/accuracies": 0.6299999952316284, + "rewards/chosen": -0.531054675579071, + "rewards/margins": 0.5484374761581421, + "rewards/rejected": -1.0789062976837158, + "step": 3550 + }, + { + "epoch": 0.9152935118434603, + "grad_norm": 510.0, + "learning_rate": 4.235324407826982e-08, + "logits/chosen": -2.559375047683716, + "logits/rejected": -2.612499952316284, + "logps/chosen": -314.6000061035156, + "logps/rejected": -378.3999938964844, + "loss": 0.4957, + "rewards/accuracies": 0.7284066081047058, + "rewards/chosen": -0.7142578363418579, + "rewards/margins": 0.870312511920929, + "rewards/rejected": -1.5859375, + "step": 3555 + }, + { + "epoch": 0.9165808444902163, + "grad_norm": 446.0, + "learning_rate": 4.170957775489186e-08, + "logits/chosen": -2.4906249046325684, + "logits/rejected": -2.565624952316284, + "logps/chosen": -345.0, + "logps/rejected": -369.20001220703125, + "loss": 0.4699, + "rewards/accuracies": 0.7746428847312927, + "rewards/chosen": -0.774951159954071, + "rewards/margins": 1.1062500476837158, + "rewards/rejected": -1.87890625, + "step": 3560 + }, + { + "epoch": 0.9178681771369722, + "grad_norm": 736.0, + "learning_rate": 4.10659114315139e-08, + "logits/chosen": -2.6656250953674316, + "logits/rejected": -2.428906202316284, + "logps/chosen": -289.0, + "logps/rejected": -316.6000061035156, + "loss": 0.5, + "rewards/accuracies": 0.7556710839271545, + "rewards/chosen": -0.22695311903953552, + "rewards/margins": 0.703125, + "rewards/rejected": -0.9296875, + "step": 3565 + }, + { + "epoch": 0.9191555097837281, + "grad_norm": 884.0, + "learning_rate": 4.042224510813594e-08, + "logits/chosen": -2.403125047683716, + "logits/rejected": -2.1015625, + "logps/chosen": -294.5, + "logps/rejected": -308.32501220703125, + "loss": 0.6035, + "rewards/accuracies": 0.7436131238937378, + "rewards/chosen": -0.6048828363418579, + "rewards/margins": 0.669921875, + "rewards/rejected": -1.2736327648162842, + "step": 3570 + }, + { + "epoch": 0.920442842430484, + "grad_norm": 760.0, + "learning_rate": 3.977857878475798e-08, + "logits/chosen": -2.543750047683716, + "logits/rejected": -2.487499952316284, + "logps/chosen": -333.20001220703125, + "logps/rejected": -306.20001220703125, + "loss": 0.5324, + "rewards/accuracies": 0.6766667366027832, + "rewards/chosen": -0.8130859136581421, + "rewards/margins": 0.8207031488418579, + "rewards/rejected": -1.6339843273162842, + "step": 3575 + }, + { + "epoch": 0.92173017507724, + "grad_norm": 928.0, + "learning_rate": 3.9134912461380015e-08, + "logits/chosen": -2.606250047683716, + "logits/rejected": -2.6781249046325684, + "logps/chosen": -293.3999938964844, + "logps/rejected": -359.6000061035156, + "loss": 0.4738, + "rewards/accuracies": 0.786794900894165, + "rewards/chosen": -0.45675772428512573, + "rewards/margins": 0.870312511920929, + "rewards/rejected": -1.325781226158142, + "step": 3580 + }, + { + "epoch": 0.9230175077239959, + "grad_norm": 430.0, + "learning_rate": 3.849124613800206e-08, + "logits/chosen": -2.668750047683716, + "logits/rejected": -2.78125, + "logps/chosen": -342.3999938964844, + "logps/rejected": -312.6000061035156, + "loss": 0.5051, + "rewards/accuracies": 0.7085714340209961, + "rewards/chosen": -0.31298828125, + "rewards/margins": 0.7808593511581421, + "rewards/rejected": -1.091796875, + "step": 3585 + }, + { + "epoch": 0.9243048403707518, + "grad_norm": 398.0, + "learning_rate": 3.7847579814624094e-08, + "logits/chosen": -2.7718749046325684, + "logits/rejected": -2.5218749046325684, + "logps/chosen": -266.3999938964844, + "logps/rejected": -256.70001220703125, + "loss": 0.5555, + "rewards/accuracies": 0.6490873098373413, + "rewards/chosen": -0.24667969346046448, + "rewards/margins": 0.640625, + "rewards/rejected": -0.8873046636581421, + "step": 3590 + }, + { + "epoch": 0.9255921730175077, + "grad_norm": 334.0, + "learning_rate": 3.720391349124614e-08, + "logits/chosen": -2.609375, + "logits/rejected": -2.3812499046325684, + "logps/chosen": -272.5, + "logps/rejected": -336.20001220703125, + "loss": 0.4625, + "rewards/accuracies": 0.7408334016799927, + "rewards/chosen": -0.490234375, + "rewards/margins": 0.970898449420929, + "rewards/rejected": -1.4609375, + "step": 3595 + }, + { + "epoch": 0.9268795056642637, + "grad_norm": 336.0, + "learning_rate": 3.656024716786817e-08, + "logits/chosen": -2.734375, + "logits/rejected": -2.6078124046325684, + "logps/chosen": -256.0, + "logps/rejected": -300.6000061035156, + "loss": 0.4227, + "rewards/accuracies": 0.7800793647766113, + "rewards/chosen": -0.1751708984375, + "rewards/margins": 1.2109375, + "rewards/rejected": -1.3859374523162842, + "step": 3600 + }, + { + "epoch": 0.9281668383110195, + "grad_norm": 482.0, + "learning_rate": 3.5916580844490216e-08, + "logits/chosen": -2.5406250953674316, + "logits/rejected": -2.311718702316284, + "logps/chosen": -286.20001220703125, + "logps/rejected": -340.0, + "loss": 0.4852, + "rewards/accuracies": 0.7224999666213989, + "rewards/chosen": -0.48552244901657104, + "rewards/margins": 0.9359375238418579, + "rewards/rejected": -1.4226562976837158, + "step": 3605 + }, + { + "epoch": 0.9294541709577755, + "grad_norm": 580.0, + "learning_rate": 3.527291452111225e-08, + "logits/chosen": -2.5875000953674316, + "logits/rejected": -2.549999952316284, + "logps/chosen": -322.0, + "logps/rejected": -372.0, + "loss": 0.4223, + "rewards/accuracies": 0.7803571820259094, + "rewards/chosen": -0.846875011920929, + "rewards/margins": 1.2449219226837158, + "rewards/rejected": -2.09375, + "step": 3610 + }, + { + "epoch": 0.9307415036045315, + "grad_norm": 1048.0, + "learning_rate": 3.4629248197734294e-08, + "logits/chosen": -2.590625047683716, + "logits/rejected": -2.6265625953674316, + "logps/chosen": -373.6000061035156, + "logps/rejected": -393.20001220703125, + "loss": 0.448, + "rewards/accuracies": 0.7547619342803955, + "rewards/chosen": -0.4764648377895355, + "rewards/margins": 1.024999976158142, + "rewards/rejected": -1.502343773841858, + "step": 3615 + }, + { + "epoch": 0.9320288362512873, + "grad_norm": 436.0, + "learning_rate": 3.398558187435633e-08, + "logits/chosen": -2.762500047683716, + "logits/rejected": -2.6781249046325684, + "logps/chosen": -273.20001220703125, + "logps/rejected": -277.5, + "loss": 0.5074, + "rewards/accuracies": 0.668181836605072, + "rewards/chosen": -0.408203125, + "rewards/margins": 0.901611328125, + "rewards/rejected": -1.3087036609649658, + "step": 3620 + }, + { + "epoch": 0.9333161688980433, + "grad_norm": 808.0, + "learning_rate": 3.334191555097837e-08, + "logits/chosen": -2.5718750953674316, + "logits/rejected": -2.578125, + "logps/chosen": -300.6000061035156, + "logps/rejected": -299.79998779296875, + "loss": 0.4727, + "rewards/accuracies": 0.7433333992958069, + "rewards/chosen": -0.5181640386581421, + "rewards/margins": 0.99609375, + "rewards/rejected": -1.513281226158142, + "step": 3625 + }, + { + "epoch": 0.9346035015447992, + "grad_norm": 324.0, + "learning_rate": 3.269824922760041e-08, + "logits/chosen": -2.6312499046325684, + "logits/rejected": -2.487499952316284, + "logps/chosen": -315.6000061035156, + "logps/rejected": -390.3999938964844, + "loss": 0.4797, + "rewards/accuracies": 0.679038405418396, + "rewards/chosen": -0.47919923067092896, + "rewards/margins": 0.8882812261581421, + "rewards/rejected": -1.365625023841858, + "step": 3630 + }, + { + "epoch": 0.935890834191555, + "grad_norm": 496.0, + "learning_rate": 3.205458290422245e-08, + "logits/chosen": -2.578125, + "logits/rejected": -2.542187452316284, + "logps/chosen": -310.6000061035156, + "logps/rejected": -322.0, + "loss": 0.6402, + "rewards/accuracies": 0.64814692735672, + "rewards/chosen": -0.6695312261581421, + "rewards/margins": 0.5902076959609985, + "rewards/rejected": -1.259374976158142, + "step": 3635 + }, + { + "epoch": 0.937178166838311, + "grad_norm": 532.0, + "learning_rate": 3.141091658084449e-08, + "logits/chosen": -2.7249999046325684, + "logits/rejected": -2.78125, + "logps/chosen": -315.0, + "logps/rejected": -294.20001220703125, + "loss": 0.6023, + "rewards/accuracies": 0.6684615612030029, + "rewards/chosen": -0.2972656190395355, + "rewards/margins": 0.49775391817092896, + "rewards/rejected": -0.7962402105331421, + "step": 3640 + }, + { + "epoch": 0.938465499485067, + "grad_norm": 544.0, + "learning_rate": 3.076725025746653e-08, + "logits/chosen": -2.6812500953674316, + "logits/rejected": -2.65625, + "logps/chosen": -391.20001220703125, + "logps/rejected": -361.20001220703125, + "loss": 0.5773, + "rewards/accuracies": 0.6714285612106323, + "rewards/chosen": -0.747851550579071, + "rewards/margins": 0.693310558795929, + "rewards/rejected": -1.443750023841858, + "step": 3645 + }, + { + "epoch": 0.9397528321318228, + "grad_norm": 1384.0, + "learning_rate": 3.0123583934088567e-08, + "logits/chosen": -2.2890625, + "logits/rejected": -2.65625, + "logps/chosen": -338.0, + "logps/rejected": -328.20001220703125, + "loss": 0.5441, + "rewards/accuracies": 0.7122862935066223, + "rewards/chosen": -0.4527831971645355, + "rewards/margins": 0.697070300579071, + "rewards/rejected": -1.15234375, + "step": 3650 + }, + { + "epoch": 0.9410401647785788, + "grad_norm": 652.0, + "learning_rate": 2.9479917610710606e-08, + "logits/chosen": -2.7249999046325684, + "logits/rejected": -2.793750047683716, + "logps/chosen": -331.20001220703125, + "logps/rejected": -369.20001220703125, + "loss": 0.473, + "rewards/accuracies": 0.7416666746139526, + "rewards/chosen": -0.3271484375, + "rewards/margins": 1.078515648841858, + "rewards/rejected": -1.40625, + "step": 3655 + }, + { + "epoch": 0.9423274974253347, + "grad_norm": 648.0, + "learning_rate": 2.8836251287332645e-08, + "logits/chosen": -2.7718749046325684, + "logits/rejected": -2.753124952316284, + "logps/chosen": -323.6000061035156, + "logps/rejected": -345.6000061035156, + "loss": 0.4414, + "rewards/accuracies": 0.7773809432983398, + "rewards/chosen": -0.27009278535842896, + "rewards/margins": 0.985156238079071, + "rewards/rejected": -1.255468726158142, + "step": 3660 + }, + { + "epoch": 0.9436148300720907, + "grad_norm": 340.0, + "learning_rate": 2.8192584963954685e-08, + "logits/chosen": -2.7406249046325684, + "logits/rejected": -2.84375, + "logps/chosen": -294.3999938964844, + "logps/rejected": -342.3999938964844, + "loss": 0.4818, + "rewards/accuracies": 0.7202020287513733, + "rewards/chosen": -0.444580078125, + "rewards/margins": 0.957812488079071, + "rewards/rejected": -1.40234375, + "step": 3665 + }, + { + "epoch": 0.9449021627188465, + "grad_norm": 756.0, + "learning_rate": 2.7548918640576724e-08, + "logits/chosen": -2.512500047683716, + "logits/rejected": -2.609375, + "logps/chosen": -282.0, + "logps/rejected": -261.3999938964844, + "loss": 0.5801, + "rewards/accuracies": 0.6966667175292969, + "rewards/chosen": -0.43603515625, + "rewards/margins": 0.558154284954071, + "rewards/rejected": -0.9957031011581421, + "step": 3670 + }, + { + "epoch": 0.9461894953656025, + "grad_norm": 832.0, + "learning_rate": 2.6905252317198764e-08, + "logits/chosen": -2.628124952316284, + "logits/rejected": -2.7281250953674316, + "logps/chosen": -250.60000610351562, + "logps/rejected": -210.6999969482422, + "loss": 0.5801, + "rewards/accuracies": 0.6719444990158081, + "rewards/chosen": -0.14140625298023224, + "rewards/margins": 0.551953136920929, + "rewards/rejected": -0.6927734613418579, + "step": 3675 + }, + { + "epoch": 0.9474768280123584, + "grad_norm": 816.0, + "learning_rate": 2.6261585993820803e-08, + "logits/chosen": -2.518749952316284, + "logits/rejected": -2.417187452316284, + "logps/chosen": -271.1000061035156, + "logps/rejected": -290.79998779296875, + "loss": 0.5855, + "rewards/accuracies": 0.6241666674613953, + "rewards/chosen": -0.533398449420929, + "rewards/margins": 0.685253918170929, + "rewards/rejected": -1.21875, + "step": 3680 + }, + { + "epoch": 0.9487641606591143, + "grad_norm": 928.0, + "learning_rate": 2.5617919670442842e-08, + "logits/chosen": -2.4781250953674316, + "logits/rejected": -2.354687452316284, + "logps/chosen": -266.0, + "logps/rejected": -333.0, + "loss": 0.5883, + "rewards/accuracies": 0.6595779657363892, + "rewards/chosen": -0.3949218690395355, + "rewards/margins": 0.745898425579071, + "rewards/rejected": -1.141210913658142, + "step": 3685 + }, + { + "epoch": 0.9500514933058702, + "grad_norm": 968.0, + "learning_rate": 2.497425334706488e-08, + "logits/chosen": -2.4437499046325684, + "logits/rejected": -2.5843749046325684, + "logps/chosen": -324.79998779296875, + "logps/rejected": -335.20001220703125, + "loss": 0.5547, + "rewards/accuracies": 0.6276923418045044, + "rewards/chosen": -0.6265624761581421, + "rewards/margins": 0.9134765863418579, + "rewards/rejected": -1.5363280773162842, + "step": 3690 + }, + { + "epoch": 0.9513388259526262, + "grad_norm": 816.0, + "learning_rate": 2.433058702368692e-08, + "logits/chosen": -2.4781250953674316, + "logits/rejected": -2.4312500953674316, + "logps/chosen": -333.20001220703125, + "logps/rejected": -375.20001220703125, + "loss": 0.5254, + "rewards/accuracies": 0.7549999952316284, + "rewards/chosen": -0.9781249761581421, + "rewards/margins": 1.03125, + "rewards/rejected": -2.010937452316284, + "step": 3695 + }, + { + "epoch": 0.952626158599382, + "grad_norm": 696.0, + "learning_rate": 2.368692070030896e-08, + "logits/chosen": -2.6031250953674316, + "logits/rejected": -2.6656250953674316, + "logps/chosen": -305.6000061035156, + "logps/rejected": -314.3999938964844, + "loss": 0.4742, + "rewards/accuracies": 0.8036364316940308, + "rewards/chosen": -0.20405273139476776, + "rewards/margins": 0.8671875, + "rewards/rejected": -1.071874976158142, + "step": 3700 + }, + { + "epoch": 0.953913491246138, + "grad_norm": 500.0, + "learning_rate": 2.3043254376931e-08, + "logits/chosen": -2.6875, + "logits/rejected": -2.6937499046325684, + "logps/chosen": -342.79998779296875, + "logps/rejected": -287.20001220703125, + "loss": 0.6055, + "rewards/accuracies": 0.5708358883857727, + "rewards/chosen": -0.5067993402481079, + "rewards/margins": 0.615039050579071, + "rewards/rejected": -1.122656226158142, + "step": 3705 + }, + { + "epoch": 0.955200823892894, + "grad_norm": 612.0, + "learning_rate": 2.239958805355304e-08, + "logits/chosen": -2.512500047683716, + "logits/rejected": -2.734375, + "logps/chosen": -265.3999938964844, + "logps/rejected": -262.6000061035156, + "loss": 0.5625, + "rewards/accuracies": 0.7104370594024658, + "rewards/chosen": -0.660449206829071, + "rewards/margins": 0.6039062738418579, + "rewards/rejected": -1.265625, + "step": 3710 + }, + { + "epoch": 0.9564881565396498, + "grad_norm": 944.0, + "learning_rate": 2.1755921730175075e-08, + "logits/chosen": -2.496875047683716, + "logits/rejected": -2.6343750953674316, + "logps/chosen": -272.20001220703125, + "logps/rejected": -254.3000030517578, + "loss": 0.607, + "rewards/accuracies": 0.6270130276679993, + "rewards/chosen": -0.4794921875, + "rewards/margins": 0.5264648199081421, + "rewards/rejected": -1.005859375, + "step": 3715 + }, + { + "epoch": 0.9577754891864058, + "grad_norm": 504.0, + "learning_rate": 2.1112255406797115e-08, + "logits/chosen": -2.53125, + "logits/rejected": -2.590625047683716, + "logps/chosen": -322.3999938964844, + "logps/rejected": -322.20001220703125, + "loss": 0.5074, + "rewards/accuracies": 0.7254762053489685, + "rewards/chosen": -0.6015625, + "rewards/margins": 1.0408203601837158, + "rewards/rejected": -1.645117163658142, + "step": 3720 + }, + { + "epoch": 0.9590628218331617, + "grad_norm": 544.0, + "learning_rate": 2.0468589083419154e-08, + "logits/chosen": -2.528125047683716, + "logits/rejected": -2.2828125953674316, + "logps/chosen": -295.6000061035156, + "logps/rejected": -352.6000061035156, + "loss": 0.6137, + "rewards/accuracies": 0.659166693687439, + "rewards/chosen": -1.1574218273162842, + "rewards/margins": 0.8291991949081421, + "rewards/rejected": -1.984375, + "step": 3725 + }, + { + "epoch": 0.9603501544799176, + "grad_norm": 460.0, + "learning_rate": 1.9824922760041193e-08, + "logits/chosen": -2.625, + "logits/rejected": -2.575000047683716, + "logps/chosen": -327.79998779296875, + "logps/rejected": -308.3999938964844, + "loss": 0.5793, + "rewards/accuracies": 0.6782143115997314, + "rewards/chosen": -0.556640625, + "rewards/margins": 0.645312488079071, + "rewards/rejected": -1.2000000476837158, + "step": 3730 + }, + { + "epoch": 0.9616374871266735, + "grad_norm": 420.0, + "learning_rate": 1.9181256436663233e-08, + "logits/chosen": -2.8218750953674316, + "logits/rejected": -2.778125047683716, + "logps/chosen": -313.3999938964844, + "logps/rejected": -324.0, + "loss": 0.6016, + "rewards/accuracies": 0.6588889360427856, + "rewards/chosen": -0.38298338651657104, + "rewards/margins": 0.5181640386581421, + "rewards/rejected": -0.8999999761581421, + "step": 3735 + }, + { + "epoch": 0.9629248197734295, + "grad_norm": 506.0, + "learning_rate": 1.8537590113285272e-08, + "logits/chosen": -2.410937547683716, + "logits/rejected": -2.440624952316284, + "logps/chosen": -221.1999969482422, + "logps/rejected": -280.3999938964844, + "loss": 0.473, + "rewards/accuracies": 0.7333333492279053, + "rewards/chosen": -0.42851561307907104, + "rewards/margins": 1.015625, + "rewards/rejected": -1.443750023841858, + "step": 3740 + }, + { + "epoch": 0.9642121524201854, + "grad_norm": 832.0, + "learning_rate": 1.789392378990731e-08, + "logits/chosen": -2.581249952316284, + "logits/rejected": -2.4781250953674316, + "logps/chosen": -285.20001220703125, + "logps/rejected": -327.20001220703125, + "loss": 0.4898, + "rewards/accuracies": 0.7884249687194824, + "rewards/chosen": -0.2918945252895355, + "rewards/margins": 1.126953125, + "rewards/rejected": -1.4210937023162842, + "step": 3745 + }, + { + "epoch": 0.9654994850669413, + "grad_norm": 462.0, + "learning_rate": 1.725025746652935e-08, + "logits/chosen": -2.589062452316284, + "logits/rejected": -2.9000000953674316, + "logps/chosen": -201.1999969482422, + "logps/rejected": -236.60000610351562, + "loss": 0.6871, + "rewards/accuracies": 0.5320635437965393, + "rewards/chosen": -0.196044921875, + "rewards/margins": 0.234375, + "rewards/rejected": -0.43095701932907104, + "step": 3750 + }, + { + "epoch": 0.9667868177136972, + "grad_norm": 486.0, + "learning_rate": 1.660659114315139e-08, + "logits/chosen": -2.331249952316284, + "logits/rejected": -2.440624952316284, + "logps/chosen": -276.79998779296875, + "logps/rejected": -356.0, + "loss": 0.407, + "rewards/accuracies": 0.8475000262260437, + "rewards/chosen": -0.681640625, + "rewards/margins": 1.2820312976837158, + "rewards/rejected": -1.96484375, + "step": 3755 + }, + { + "epoch": 0.9680741503604532, + "grad_norm": 236.0, + "learning_rate": 1.596292481977343e-08, + "logits/chosen": -2.40625, + "logits/rejected": -2.378124952316284, + "logps/chosen": -304.79998779296875, + "logps/rejected": -365.20001220703125, + "loss": 0.3961, + "rewards/accuracies": 0.8208333849906921, + "rewards/chosen": -0.8511718511581421, + "rewards/margins": 1.404687523841858, + "rewards/rejected": -2.2578125, + "step": 3760 + }, + { + "epoch": 0.969361483007209, + "grad_norm": 2288.0, + "learning_rate": 1.531925849639547e-08, + "logits/chosen": -2.526562452316284, + "logits/rejected": -2.3843750953674316, + "logps/chosen": -293.20001220703125, + "logps/rejected": -294.79998779296875, + "loss": 0.6273, + "rewards/accuracies": 0.7011111378669739, + "rewards/chosen": -0.42241209745407104, + "rewards/margins": 0.6451171636581421, + "rewards/rejected": -1.0662109851837158, + "step": 3765 + }, + { + "epoch": 0.970648815653965, + "grad_norm": 716.0, + "learning_rate": 1.4675592173017507e-08, + "logits/chosen": -2.528125047683716, + "logits/rejected": -2.145312547683716, + "logps/chosen": -276.29998779296875, + "logps/rejected": -334.6000061035156, + "loss": 0.5312, + "rewards/accuracies": 0.7208333611488342, + "rewards/chosen": -0.517041027545929, + "rewards/margins": 0.991015613079071, + "rewards/rejected": -1.509179711341858, + "step": 3770 + }, + { + "epoch": 0.971936148300721, + "grad_norm": 376.0, + "learning_rate": 1.4031925849639546e-08, + "logits/chosen": -2.6781249046325684, + "logits/rejected": -2.75, + "logps/chosen": -328.6000061035156, + "logps/rejected": -321.0, + "loss": 0.5371, + "rewards/accuracies": 0.7106685638427734, + "rewards/chosen": -0.0069213868118822575, + "rewards/margins": 0.6609161496162415, + "rewards/rejected": -0.6679443120956421, + "step": 3775 + }, + { + "epoch": 0.9732234809474768, + "grad_norm": 398.0, + "learning_rate": 1.3388259526261585e-08, + "logits/chosen": -2.628124952316284, + "logits/rejected": -2.5406250953674316, + "logps/chosen": -290.0, + "logps/rejected": -339.3999938964844, + "loss": 0.4332, + "rewards/accuracies": 0.7439435720443726, + "rewards/chosen": -0.22700805962085724, + "rewards/margins": 0.917187511920929, + "rewards/rejected": -1.1453125476837158, + "step": 3780 + }, + { + "epoch": 0.9745108135942327, + "grad_norm": 760.0, + "learning_rate": 1.2744593202883625e-08, + "logits/chosen": -2.6156249046325684, + "logits/rejected": -2.731250047683716, + "logps/chosen": -317.0, + "logps/rejected": -292.3999938964844, + "loss": 0.5992, + "rewards/accuracies": 0.68376624584198, + "rewards/chosen": -0.35734862089157104, + "rewards/margins": 0.653759777545929, + "rewards/rejected": -1.0110352039337158, + "step": 3785 + }, + { + "epoch": 0.9757981462409887, + "grad_norm": 564.0, + "learning_rate": 1.2100926879505664e-08, + "logits/chosen": -2.596874952316284, + "logits/rejected": -2.753124952316284, + "logps/chosen": -252.14999389648438, + "logps/rejected": -291.3500061035156, + "loss": 0.5367, + "rewards/accuracies": 0.7703571319580078, + "rewards/chosen": -0.29887694120407104, + "rewards/margins": 0.774218738079071, + "rewards/rejected": -1.0734374523162842, + "step": 3790 + }, + { + "epoch": 0.9770854788877446, + "grad_norm": 940.0, + "learning_rate": 1.1457260556127703e-08, + "logits/chosen": -2.528125047683716, + "logits/rejected": -2.703125, + "logps/chosen": -271.3999938964844, + "logps/rejected": -350.79998779296875, + "loss": 0.4605, + "rewards/accuracies": 0.7516549825668335, + "rewards/chosen": -0.49348145723342896, + "rewards/margins": 1.036718726158142, + "rewards/rejected": -1.53125, + "step": 3795 + }, + { + "epoch": 0.9783728115345005, + "grad_norm": 600.0, + "learning_rate": 1.0813594232749741e-08, + "logits/chosen": -2.5687499046325684, + "logits/rejected": -2.5250000953674316, + "logps/chosen": -338.6000061035156, + "logps/rejected": -379.20001220703125, + "loss": 0.4988, + "rewards/accuracies": 0.7018589973449707, + "rewards/chosen": -0.760546863079071, + "rewards/margins": 1.0945312976837158, + "rewards/rejected": -1.853124976158142, + "step": 3800 + }, + { + "epoch": 0.9796601441812565, + "grad_norm": 482.0, + "learning_rate": 1.016992790937178e-08, + "logits/chosen": -2.546875, + "logits/rejected": -2.503124952316284, + "logps/chosen": -259.0, + "logps/rejected": -379.3999938964844, + "loss": 0.4008, + "rewards/accuracies": 0.7895238399505615, + "rewards/chosen": -0.814990222454071, + "rewards/margins": 1.4484374523162842, + "rewards/rejected": -2.262500047683716, + "step": 3805 + }, + { + "epoch": 0.9809474768280123, + "grad_norm": 732.0, + "learning_rate": 9.52626158599382e-09, + "logits/chosen": -2.59375, + "logits/rejected": -2.6312499046325684, + "logps/chosen": -281.20001220703125, + "logps/rejected": -283.79998779296875, + "loss": 0.6969, + "rewards/accuracies": 0.5841666460037231, + "rewards/chosen": -0.6783202886581421, + "rewards/margins": 0.49433594942092896, + "rewards/rejected": -1.171484351158142, + "step": 3810 + }, + { + "epoch": 0.9822348094747683, + "grad_norm": 624.0, + "learning_rate": 8.88259526261586e-09, + "logits/chosen": -2.715625047683716, + "logits/rejected": -2.799999952316284, + "logps/chosen": -258.79998779296875, + "logps/rejected": -299.6000061035156, + "loss": 0.6195, + "rewards/accuracies": 0.5803571939468384, + "rewards/chosen": -0.5238281488418579, + "rewards/margins": 0.5287109613418579, + "rewards/rejected": -1.0517089366912842, + "step": 3815 + }, + { + "epoch": 0.9835221421215242, + "grad_norm": 668.0, + "learning_rate": 8.238928939237899e-09, + "logits/chosen": -2.581249952316284, + "logits/rejected": -2.549999952316284, + "logps/chosen": -296.79998779296875, + "logps/rejected": -359.3999938964844, + "loss": 0.643, + "rewards/accuracies": 0.5970238447189331, + "rewards/chosen": -0.601269543170929, + "rewards/margins": 0.586181640625, + "rewards/rejected": -1.1847655773162842, + "step": 3820 + }, + { + "epoch": 0.9848094747682801, + "grad_norm": 660.0, + "learning_rate": 7.595262615859938e-09, + "logits/chosen": -2.40625, + "logits/rejected": -2.453125, + "logps/chosen": -289.20001220703125, + "logps/rejected": -392.79998779296875, + "loss": 0.5078, + "rewards/accuracies": 0.7225000262260437, + "rewards/chosen": -0.8617187738418579, + "rewards/margins": 1.003515601158142, + "rewards/rejected": -1.865625023841858, + "step": 3825 + }, + { + "epoch": 0.986096807415036, + "grad_norm": 628.0, + "learning_rate": 6.951596292481977e-09, + "logits/chosen": -2.575000047683716, + "logits/rejected": -2.621875047683716, + "logps/chosen": -350.3999938964844, + "logps/rejected": -386.0, + "loss": 0.5031, + "rewards/accuracies": 0.6193181872367859, + "rewards/chosen": -0.64892578125, + "rewards/margins": 1.0402343273162842, + "rewards/rejected": -1.692968726158142, + "step": 3830 + }, + { + "epoch": 0.987384140061792, + "grad_norm": 624.0, + "learning_rate": 6.307929969104016e-09, + "logits/chosen": -2.578125, + "logits/rejected": -2.239062547683716, + "logps/chosen": -352.3999938964844, + "logps/rejected": -424.3999938964844, + "loss": 0.5473, + "rewards/accuracies": 0.6299999952316284, + "rewards/chosen": -1.087499976158142, + "rewards/margins": 0.9688720703125, + "rewards/rejected": -2.059375047683716, + "step": 3835 + }, + { + "epoch": 0.9886714727085479, + "grad_norm": 544.0, + "learning_rate": 5.664263645726055e-09, + "logits/chosen": -2.59375, + "logits/rejected": -2.7406249046325684, + "logps/chosen": -359.6000061035156, + "logps/rejected": -356.3999938964844, + "loss": 0.5539, + "rewards/accuracies": 0.6569697260856628, + "rewards/chosen": -0.2870117127895355, + "rewards/margins": 0.6898437738418579, + "rewards/rejected": -0.9769531488418579, + "step": 3840 + }, + { + "epoch": 0.9899588053553038, + "grad_norm": 430.0, + "learning_rate": 5.020597322348095e-09, + "logits/chosen": -2.581249952316284, + "logits/rejected": -2.784374952316284, + "logps/chosen": -378.0, + "logps/rejected": -364.79998779296875, + "loss": 0.4437, + "rewards/accuracies": 0.7289285659790039, + "rewards/chosen": -0.16826172173023224, + "rewards/margins": 0.953906238079071, + "rewards/rejected": -1.1238281726837158, + "step": 3845 + }, + { + "epoch": 0.9912461380020597, + "grad_norm": 484.0, + "learning_rate": 4.376930998970134e-09, + "logits/chosen": -2.762500047683716, + "logits/rejected": -2.6968750953674316, + "logps/chosen": -364.0, + "logps/rejected": -345.20001220703125, + "loss": 0.5289, + "rewards/accuracies": 0.691529393196106, + "rewards/chosen": -0.49455565214157104, + "rewards/margins": 0.8091796636581421, + "rewards/rejected": -1.303125023841858, + "step": 3850 + }, + { + "epoch": 0.9925334706488157, + "grad_norm": 342.0, + "learning_rate": 3.733264675592173e-09, + "logits/chosen": -2.512500047683716, + "logits/rejected": -2.2601561546325684, + "logps/chosen": -270.20001220703125, + "logps/rejected": -287.6000061035156, + "loss": 0.5328, + "rewards/accuracies": 0.7850000858306885, + "rewards/chosen": -0.684765636920929, + "rewards/margins": 0.709912121295929, + "rewards/rejected": -1.3937499523162842, + "step": 3855 + }, + { + "epoch": 0.9938208032955715, + "grad_norm": 620.0, + "learning_rate": 3.089598352214212e-09, + "logits/chosen": -2.621875047683716, + "logits/rejected": -2.6500000953674316, + "logps/chosen": -362.79998779296875, + "logps/rejected": -393.6000061035156, + "loss": 0.5406, + "rewards/accuracies": 0.73333340883255, + "rewards/chosen": -0.876953125, + "rewards/margins": 0.9609375, + "rewards/rejected": -1.841406226158142, + "step": 3860 + }, + { + "epoch": 0.9951081359423275, + "grad_norm": 432.0, + "learning_rate": 2.445932028836251e-09, + "logits/chosen": -2.464062452316284, + "logits/rejected": -2.5999999046325684, + "logps/chosen": -365.20001220703125, + "logps/rejected": -408.3999938964844, + "loss": 0.498, + "rewards/accuracies": 0.7125000357627869, + "rewards/chosen": -0.7216796875, + "rewards/margins": 0.9554687738418579, + "rewards/rejected": -1.6765625476837158, + "step": 3865 + }, + { + "epoch": 0.9963954685890835, + "grad_norm": 486.0, + "learning_rate": 1.8022657054582903e-09, + "logits/chosen": -2.5687499046325684, + "logits/rejected": -2.5093750953674316, + "logps/chosen": -317.6000061035156, + "logps/rejected": -405.6000061035156, + "loss": 0.4516, + "rewards/accuracies": 0.7634615302085876, + "rewards/chosen": -0.6068359613418579, + "rewards/margins": 1.2000000476837158, + "rewards/rejected": -1.806249976158142, + "step": 3870 + }, + { + "epoch": 0.9976828012358393, + "grad_norm": 752.0, + "learning_rate": 1.1585993820803295e-09, + "logits/chosen": -2.6312499046325684, + "logits/rejected": -2.721874952316284, + "logps/chosen": -332.0, + "logps/rejected": -317.3999938964844, + "loss": 0.5625, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.37626951932907104, + "rewards/margins": 0.735156238079071, + "rewards/rejected": -1.1124999523162842, + "step": 3875 + }, + { + "epoch": 0.9989701338825953, + "grad_norm": 480.0, + "learning_rate": 5.149330587023687e-10, + "logits/chosen": -2.6781249046325684, + "logits/rejected": -2.4593749046325684, + "logps/chosen": -301.79998779296875, + "logps/rejected": -264.79998779296875, + "loss": 0.5188, + "rewards/accuracies": 0.6945163011550903, + "rewards/chosen": -0.31562501192092896, + "rewards/margins": 0.633984386920929, + "rewards/rejected": -0.948437511920929, + "step": 3880 + } + ], + "logging_steps": 5, + "max_steps": 3884, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 3000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +}