diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,4042 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 250,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004,
+      "grad_norm": 108.45250504320283,
+      "learning_rate": 1.25e-07,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -1.1796875,
+      "logps/chosen": -32.0,
+      "logps/rejected": -35.25,
+      "loss": 4.0839,
+      "nll_loss": 3.421875,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.04150390625,
+      "rewards/margins": 0.099609375,
+      "rewards/rejected": -0.058349609375,
+      "step": 1
+    },
+    {
+      "epoch": 0.008,
+      "grad_norm": 120.95987737750156,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": -1.0703125,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -38.0,
+      "logps/rejected": -35.25,
+      "loss": 4.3121,
+      "nll_loss": 3.984375,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": 0.046142578125,
+      "rewards/margins": 0.01171875,
+      "rewards/rejected": 0.034423828125,
+      "step": 2
+    },
+    {
+      "epoch": 0.012,
+      "grad_norm": 109.89014011391664,
+      "learning_rate": 3.75e-07,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -1.171875,
+      "logps/chosen": -33.0,
+      "logps/rejected": -37.0,
+      "loss": 4.2,
+      "nll_loss": 3.3125,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.05859375,
+      "rewards/margins": 0.07421875,
+      "rewards/rejected": -0.0155029296875,
+      "step": 3
+    },
+    {
+      "epoch": 0.016,
+      "grad_norm": 105.87845131816027,
+      "learning_rate": 5e-07,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -1.203125,
+      "logps/chosen": -31.25,
+      "logps/rejected": -36.75,
+      "loss": 4.2118,
+      "nll_loss": 3.03125,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0595703125,
+      "rewards/margins": 0.0390625,
+      "rewards/rejected": 0.020263671875,
+      "step": 4
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 112.81877726787484,
+      "learning_rate": 6.249999999999999e-07,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -1.2109375,
+      "logps/chosen": -33.0,
+      "logps/rejected": -34.25,
+      "loss": 4.0945,
+      "nll_loss": 3.453125,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 0.06298828125,
+      "rewards/margins": -0.00994873046875,
+      "rewards/rejected": 0.07275390625,
+      "step": 5
+    },
+    {
+      "epoch": 0.024,
+      "grad_norm": 106.29693927531781,
+      "learning_rate": 7.5e-07,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -1.1875,
+      "logps/chosen": -33.0,
+      "logps/rejected": -36.5,
+      "loss": 4.1827,
+      "nll_loss": 3.3125,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 0.08642578125,
+      "rewards/margins": 0.037109375,
+      "rewards/rejected": 0.04931640625,
+      "step": 6
+    },
+    {
+      "epoch": 0.028,
+      "grad_norm": 114.94478205565038,
+      "learning_rate": 8.75e-07,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -1.171875,
+      "logps/chosen": -27.875,
+      "logps/rejected": -33.5,
+      "loss": 4.1499,
+      "nll_loss": 2.84375,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0625,
+      "rewards/margins": 0.01165771484375,
+      "rewards/rejected": 0.051025390625,
+      "step": 7
+    },
+    {
+      "epoch": 0.032,
+      "grad_norm": 97.37219322957407,
+      "learning_rate": 1e-06,
+      "logits/chosen": -1.171875,
+      "logits/rejected": -1.171875,
+      "logps/chosen": -32.5,
+      "logps/rejected": -30.625,
+      "loss": 4.0641,
+      "nll_loss": 3.515625,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.1552734375,
+      "rewards/margins": 0.030517578125,
+      "rewards/rejected": 0.125,
+      "step": 8
+    },
+    {
+      "epoch": 0.036,
+      "grad_norm": 108.05293741004135,
+      "learning_rate": 9.999578688879084e-07,
+      "logits/chosen": -1.15625,
+      "logits/rejected": -1.1484375,
+      "logps/chosen": -31.875,
+      "logps/rejected": -37.25,
+      "loss": 4.2044,
+      "nll_loss": 3.234375,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.19140625,
+      "rewards/margins": 0.055908203125,
+      "rewards/rejected": 0.134765625,
+      "step": 9
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 89.47841349842803,
+      "learning_rate": 9.998314826517563e-07,
+      "logits/chosen": -1.1015625,
+      "logits/rejected": -1.1171875,
+      "logps/chosen": -27.75,
+      "logps/rejected": -29.125,
+      "loss": 3.8131,
+      "nll_loss": 2.875,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": 0.318359375,
+      "rewards/margins": -0.049072265625,
+      "rewards/rejected": 0.3671875,
+      "step": 10
+    },
+    {
+      "epoch": 0.044,
+      "grad_norm": 82.2909196761966,
+      "learning_rate": 9.99620862590714e-07,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -1.1640625,
+      "logps/chosen": -26.0,
+      "logps/rejected": -30.625,
+      "loss": 3.7873,
+      "nll_loss": 2.640625,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.5390625,
+      "rewards/margins": 0.028076171875,
+      "rewards/rejected": 0.51171875,
+      "step": 11
+    },
+    {
+      "epoch": 0.048,
+      "grad_norm": 77.5417011003063,
+      "learning_rate": 9.993260441994114e-07,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -1.1328125,
+      "logps/chosen": -26.375,
+      "logps/rejected": -25.125,
+      "loss": 3.6977,
+      "nll_loss": 2.8125,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.515625,
+      "rewards/margins": 0.02099609375,
+      "rewards/rejected": 0.49609375,
+      "step": 12
+    },
+    {
+      "epoch": 0.052,
+      "grad_norm": 77.36031138542103,
+      "learning_rate": 9.989470771619553e-07,
+      "logits/chosen": -1.09375,
+      "logits/rejected": -1.109375,
+      "logps/chosen": -31.375,
+      "logps/rejected": -31.75,
+      "loss": 3.84,
+      "nll_loss": 3.171875,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.66015625,
+      "rewards/margins": 0.0380859375,
+      "rewards/rejected": 0.62109375,
+      "step": 13
+    },
+    {
+      "epoch": 0.056,
+      "grad_norm": 38.474184849981015,
+      "learning_rate": 9.984840253435568e-07,
+      "logits/chosen": -1.1640625,
+      "logits/rejected": -1.140625,
+      "logps/chosen": -21.5,
+      "logps/rejected": -23.125,
+      "loss": 2.9964,
+      "nll_loss": 2.15625,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 1.0625,
+      "rewards/margins": 0.07958984375,
+      "rewards/rejected": 0.984375,
+      "step": 14
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 37.90903240520294,
+      "learning_rate": 9.979369667797674e-07,
+      "logits/chosen": -1.0703125,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -20.5,
+      "logps/rejected": -25.25,
+      "loss": 3.0721,
+      "nll_loss": 2.0,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 1.078125,
+      "rewards/margins": -0.060546875,
+      "rewards/rejected": 1.140625,
+      "step": 15
+    },
+    {
+      "epoch": 0.064,
+      "grad_norm": 35.90082325141723,
+      "learning_rate": 9.973059936633306e-07,
+      "logits/chosen": -1.078125,
+      "logits/rejected": -1.1015625,
+      "logps/chosen": -17.0,
+      "logps/rejected": -25.25,
+      "loss": 2.9806,
+      "nll_loss": 1.71875,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.875,
+      "rewards/margins": -0.0517578125,
+      "rewards/rejected": 0.92578125,
+      "step": 16
+    },
+    {
+      "epoch": 0.068,
+      "grad_norm": 34.39419041472641,
+      "learning_rate": 9.965912123286424e-07,
+      "logits/chosen": -0.99609375,
+      "logits/rejected": -0.98046875,
+      "logps/chosen": -21.875,
+      "logps/rejected": -24.375,
+      "loss": 2.9524,
+      "nll_loss": 2.125,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 1.109375,
+      "rewards/margins": 0.1552734375,
+      "rewards/rejected": 0.95703125,
+      "step": 17
+    },
+    {
+      "epoch": 0.072,
+      "grad_norm": 33.10304291271997,
+      "learning_rate": 9.95792743233833e-07,
+      "logits/chosen": -1.03125,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -18.75,
+      "logps/rejected": -24.25,
+      "loss": 2.8946,
+      "nll_loss": 1.9765625,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 1.2265625,
+      "rewards/margins": 0.0380859375,
+      "rewards/rejected": 1.1875,
+      "step": 18
+    },
+    {
+      "epoch": 0.076,
+      "grad_norm": 28.411495189507995,
+      "learning_rate": 9.949107209404663e-07,
+      "logits/chosen": -1.03125,
+      "logits/rejected": -1.0234375,
+      "logps/chosen": -19.5,
+      "logps/rejected": -22.875,
+      "loss": 2.7523,
+      "nll_loss": 1.984375,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 1.5703125,
+      "rewards/margins": 0.0096435546875,
+      "rewards/rejected": 1.5625,
+      "step": 19
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 30.85467104218471,
+      "learning_rate": 9.939452940908626e-07,
+      "logits/chosen": -1.0703125,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -20.875,
+      "logps/rejected": -23.25,
+      "loss": 2.8015,
+      "nll_loss": 2.0625,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 1.6171875,
+      "rewards/margins": 0.296875,
+      "rewards/rejected": 1.3203125,
+      "step": 20
+    },
+    {
+      "epoch": 0.084,
+      "grad_norm": 20.81104325101837,
+      "learning_rate": 9.92896625383049e-07,
+      "logits/chosen": -0.98828125,
+      "logits/rejected": -0.9765625,
+      "logps/chosen": -16.125,
+      "logps/rejected": -21.0,
+      "loss": 2.6257,
+      "nll_loss": 1.640625,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 1.4609375,
+      "rewards/margins": -0.09423828125,
+      "rewards/rejected": 1.5546875,
+      "step": 21
+    },
+    {
+      "epoch": 0.088,
+      "grad_norm": 16.555834400457428,
+      "learning_rate": 9.917648915433411e-07,
+      "logits/chosen": -0.95703125,
+      "logits/rejected": -0.94140625,
+      "logps/chosen": -15.8125,
+      "logps/rejected": -16.875,
+      "loss": 2.4551,
+      "nll_loss": 1.53125,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 1.703125,
+      "rewards/margins": 0.16796875,
+      "rewards/rejected": 1.53125,
+      "step": 22
+    },
+    {
+      "epoch": 0.092,
+      "grad_norm": 15.376428724494609,
+      "learning_rate": 9.905502832965603e-07,
+      "logits/chosen": -0.9296875,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -15.4375,
+      "logps/rejected": -18.5,
+      "loss": 2.4073,
+      "nll_loss": 1.546875,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 1.6640625,
+      "rewards/margins": -0.029052734375,
+      "rewards/rejected": 1.6953125,
+      "step": 23
+    },
+    {
+      "epoch": 0.096,
+      "grad_norm": 16.237442660961605,
+      "learning_rate": 9.892530053338907e-07,
+      "logits/chosen": -0.89453125,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -15.1875,
+      "logps/rejected": -19.75,
+      "loss": 2.3768,
+      "nll_loss": 1.5546875,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 1.875,
+      "rewards/margins": 0.08251953125,
+      "rewards/rejected": 1.7890625,
+      "step": 24
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 15.758858425294349,
+      "learning_rate": 9.87873276278386e-07,
+      "logits/chosen": -0.8984375,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -14.5625,
+      "logps/rejected": -16.875,
+      "loss": 2.3852,
+      "nll_loss": 1.4453125,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 2.125,
+      "rewards/margins": 0.05029296875,
+      "rewards/rejected": 2.078125,
+      "step": 25
+    },
+    {
+      "epoch": 0.104,
+      "grad_norm": 16.347740539403798,
+      "learning_rate": 9.864113286481236e-07,
+      "logits/chosen": -0.9453125,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -14.125,
+      "logps/rejected": -20.0,
+      "loss": 2.317,
+      "nll_loss": 1.46875,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 1.828125,
+      "rewards/margins": -0.001220703125,
+      "rewards/rejected": 1.828125,
+      "step": 26
+    },
+    {
+      "epoch": 0.108,
+      "grad_norm": 15.099343265144629,
+      "learning_rate": 9.84867408817022e-07,
+      "logits/chosen": -0.95703125,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -13.125,
+      "logps/rejected": -16.625,
+      "loss": 2.2059,
+      "nll_loss": 1.328125,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 2.015625,
+      "rewards/margins": 0.1494140625,
+      "rewards/rejected": 1.859375,
+      "step": 27
+    },
+    {
+      "epoch": 0.112,
+      "grad_norm": 15.421632558925108,
+      "learning_rate": 9.832417769733185e-07,
+      "logits/chosen": -0.94140625,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -13.3125,
+      "logps/rejected": -17.0,
+      "loss": 2.2687,
+      "nll_loss": 1.2578125,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 1.890625,
+      "rewards/margins": 0.044921875,
+      "rewards/rejected": 1.84375,
+      "step": 28
+    },
+    {
+      "epoch": 0.116,
+      "grad_norm": 15.06033701313281,
+      "learning_rate": 9.815347070757231e-07,
+      "logits/chosen": -0.97265625,
+      "logits/rejected": -0.9609375,
+      "logps/chosen": -14.875,
+      "logps/rejected": -17.875,
+      "loss": 2.2442,
+      "nll_loss": 1.375,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 2.109375,
+      "rewards/margins": 0.3984375,
+      "rewards/rejected": 1.71875,
+      "step": 29
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 13.532947619496891,
+      "learning_rate": 9.797464868072486e-07,
+      "logits/chosen": -0.91796875,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -13.0625,
+      "logps/rejected": -16.75,
+      "loss": 2.1401,
+      "nll_loss": 1.3125,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 1.8203125,
+      "rewards/margins": 0.18359375,
+      "rewards/rejected": 1.6328125,
+      "step": 30
+    },
+    {
+      "epoch": 0.124,
+      "grad_norm": 13.807093125286677,
+      "learning_rate": 9.778774175267295e-07,
+      "logits/chosen": -0.9375,
+      "logits/rejected": -0.9296875,
+      "logps/chosen": -11.75,
+      "logps/rejected": -17.75,
+      "loss": 2.1347,
+      "nll_loss": 1.1953125,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 2.109375,
+      "rewards/margins": 0.1552734375,
+      "rewards/rejected": 1.9453125,
+      "step": 31
+    },
+    {
+      "epoch": 0.128,
+      "grad_norm": 14.709867281760857,
+      "learning_rate": 9.759278142180347e-07,
+      "logits/chosen": -0.9765625,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -9.8125,
+      "logps/rejected": -15.3125,
+      "loss": 2.0944,
+      "nll_loss": 0.98828125,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 1.96875,
+      "rewards/margins": 0.11962890625,
+      "rewards/rejected": 1.8515625,
+      "step": 32
+    },
+    {
+      "epoch": 0.132,
+      "grad_norm": 12.980725519975202,
+      "learning_rate": 9.73898005436987e-07,
+      "logits/chosen": -0.95703125,
+      "logits/rejected": -0.96484375,
+      "logps/chosen": -11.0,
+      "logps/rejected": -14.8125,
+      "loss": 2.032,
+      "nll_loss": 1.078125,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 1.9921875,
+      "rewards/margins": 0.345703125,
+      "rewards/rejected": 1.6484375,
+      "step": 33
+    },
+    {
+      "epoch": 0.136,
+      "grad_norm": 12.423187407094442,
+      "learning_rate": 9.71788333255991e-07,
+      "logits/chosen": -0.97265625,
+      "logits/rejected": -0.98046875,
+      "logps/chosen": -12.0625,
+      "logps/rejected": -15.125,
+      "loss": 1.9855,
+      "nll_loss": 1.2890625,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 2.125,
+      "rewards/margins": 0.392578125,
+      "rewards/rejected": 1.734375,
+      "step": 34
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 10.78340956295933,
+      "learning_rate": 9.695991532063874e-07,
+      "logits/chosen": -1.0078125,
+      "logits/rejected": -0.97265625,
+      "logps/chosen": -11.5,
+      "logps/rejected": -14.5625,
+      "loss": 1.9101,
+      "nll_loss": 1.1484375,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.53125,
+      "rewards/margins": 0.71875,
+      "rewards/rejected": 1.8125,
+      "step": 35
+    },
+    {
+      "epoch": 0.144,
+      "grad_norm": 10.539848066893843,
+      "learning_rate": 9.673308342185365e-07,
+      "logits/chosen": -0.9609375,
+      "logits/rejected": -0.9609375,
+      "logps/chosen": -11.6875,
+      "logps/rejected": -16.625,
+      "loss": 1.8997,
+      "nll_loss": 1.1640625,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 2.21875,
+      "rewards/margins": 0.224609375,
+      "rewards/rejected": 1.984375,
+      "step": 36
+    },
+    {
+      "epoch": 0.148,
+      "grad_norm": 10.7200500702218,
+      "learning_rate": 9.649837585596444e-07,
+      "logits/chosen": -0.921875,
+      "logits/rejected": -0.9296875,
+      "logps/chosen": -10.25,
+      "logps/rejected": -16.125,
+      "loss": 1.8621,
+      "nll_loss": 0.94921875,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 2.171875,
+      "rewards/margins": 0.345703125,
+      "rewards/rejected": 1.828125,
+      "step": 37
+    },
+    {
+      "epoch": 0.152,
+      "grad_norm": 9.54892533874284,
+      "learning_rate": 9.625583217693418e-07,
+      "logits/chosen": -0.93359375,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -11.125,
+      "logps/rejected": -14.375,
+      "loss": 1.7924,
+      "nll_loss": 1.0703125,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 3.09375,
+      "rewards/margins": 1.09375,
+      "rewards/rejected": 1.9921875,
+      "step": 38
+    },
+    {
+      "epoch": 0.156,
+      "grad_norm": 10.315046477409297,
+      "learning_rate": 9.600549325930259e-07,
+      "logits/chosen": -0.921875,
+      "logits/rejected": -0.92578125,
+      "logps/chosen": -10.375,
+      "logps/rejected": -15.5,
+      "loss": 1.8186,
+      "nll_loss": 1.0546875,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 2.5625,
+      "rewards/margins": 0.51171875,
+      "rewards/rejected": 2.046875,
+      "step": 39
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 8.518386491338127,
+      "learning_rate": 9.574740129129765e-07,
+      "logits/chosen": -0.92578125,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -10.8125,
+      "logps/rejected": -14.4375,
+      "loss": 1.7032,
+      "nll_loss": 1.0234375,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 2.015625,
+      "rewards/margins": 0.07958984375,
+      "rewards/rejected": 1.9375,
+      "step": 40
+    },
+    {
+      "epoch": 0.164,
+      "grad_norm": 9.17761100715551,
+      "learning_rate": 9.548159976772592e-07,
+      "logits/chosen": -0.95703125,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -9.375,
+      "logps/rejected": -14.375,
+      "loss": 1.8061,
+      "nll_loss": 0.93359375,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 2.4375,
+      "rewards/margins": 0.357421875,
+      "rewards/rejected": 2.09375,
+      "step": 41
+    },
+    {
+      "epoch": 0.168,
+      "grad_norm": 9.6427558562046,
+      "learning_rate": 9.520813348264251e-07,
+      "logits/chosen": -0.91015625,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -8.6875,
+      "logps/rejected": -20.75,
+      "loss": 1.6474,
+      "nll_loss": 0.82421875,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 2.8125,
+      "rewards/margins": 0.98828125,
+      "rewards/rejected": 1.8203125,
+      "step": 42
+    },
+    {
+      "epoch": 0.172,
+      "grad_norm": 10.106867319730664,
+      "learning_rate": 9.492704852180226e-07,
+      "logits/chosen": -0.88671875,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -8.0625,
+      "logps/rejected": -17.625,
+      "loss": 1.6427,
+      "nll_loss": 0.734375,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 2.609375,
+      "rewards/margins": 0.9296875,
+      "rewards/rejected": 1.671875,
+      "step": 43
+    },
+    {
+      "epoch": 0.176,
+      "grad_norm": 9.780716883236895,
+      "learning_rate": 9.463839225489318e-07,
+      "logits/chosen": -0.91015625,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -9.5,
+      "logps/rejected": -15.375,
+      "loss": 1.7142,
+      "nll_loss": 0.93359375,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 2.625,
+      "rewards/margins": 0.515625,
+      "rewards/rejected": 2.109375,
+      "step": 44
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 9.324567213910779,
+      "learning_rate": 9.434221332755338e-07,
+      "logits/chosen": -0.90234375,
+      "logits/rejected": -0.88671875,
+      "logps/chosen": -8.125,
+      "logps/rejected": -12.125,
+      "loss": 1.6251,
+      "nll_loss": 0.81640625,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 2.515625,
+      "rewards/margins": 0.138671875,
+      "rewards/rejected": 2.375,
+      "step": 45
+    },
+    {
+      "epoch": 0.184,
+      "grad_norm": 9.236623014788846,
+      "learning_rate": 9.403856165317321e-07,
+      "logits/chosen": -0.91015625,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -8.875,
+      "logps/rejected": -18.75,
+      "loss": 1.542,
+      "nll_loss": 0.87890625,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 2.515625,
+      "rewards/margins": 0.6796875,
+      "rewards/rejected": 1.828125,
+      "step": 46
+    },
+    {
+      "epoch": 0.188,
+      "grad_norm": 9.078752548744747,
+      "learning_rate": 9.372748840448361e-07,
+      "logits/chosen": -0.8828125,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -6.3125,
+      "logps/rejected": -19.25,
+      "loss": 1.5166,
+      "nll_loss": 0.62890625,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.375,
+      "rewards/margins": 0.68359375,
+      "rewards/rejected": 1.6875,
+      "step": 47
+    },
+    {
+      "epoch": 0.192,
+      "grad_norm": 9.007283454701437,
+      "learning_rate": 9.340904600493219e-07,
+      "logits/chosen": -0.890625,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -7.15625,
+      "logps/rejected": -17.125,
+      "loss": 1.5793,
+      "nll_loss": 0.78125,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 2.421875,
+      "rewards/margins": 0.55078125,
+      "rewards/rejected": 1.8671875,
+      "step": 48
+    },
+    {
+      "epoch": 0.196,
+      "grad_norm": 9.111837952613337,
+      "learning_rate": 9.308328811984869e-07,
+      "logits/chosen": -0.92578125,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -7.375,
+      "logps/rejected": -19.25,
+      "loss": 1.4635,
+      "nll_loss": 0.6796875,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.4375,
+      "rewards/margins": 0.90625,
+      "rewards/rejected": 1.5390625,
+      "step": 49
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 8.886170629930112,
+      "learning_rate": 9.275026964740101e-07,
+      "logits/chosen": -0.93359375,
+      "logits/rejected": -0.9296875,
+      "logps/chosen": -7.25,
+      "logps/rejected": -14.3125,
+      "loss": 1.4143,
+      "nll_loss": 0.6796875,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 2.65625,
+      "rewards/margins": 0.474609375,
+      "rewards/rejected": 2.171875,
+      "step": 50
+    },
+    {
+      "epoch": 0.204,
+      "grad_norm": 8.951054512095832,
+      "learning_rate": 9.241004670934347e-07,
+      "logits/chosen": -0.9140625,
+      "logits/rejected": -0.88671875,
+      "logps/chosen": -6.46875,
+      "logps/rejected": -18.625,
+      "loss": 1.4843,
+      "nll_loss": 0.59375,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.5,
+      "rewards/margins": 0.78515625,
+      "rewards/rejected": 1.71875,
+      "step": 51
+    },
+    {
+      "epoch": 0.208,
+      "grad_norm": 8.428410886373113,
+      "learning_rate": 9.206267664155906e-07,
+      "logits/chosen": -0.9140625,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -8.5625,
+      "logps/rejected": -17.0,
+      "loss": 1.5279,
+      "nll_loss": 0.875,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 2.59375,
+      "rewards/margins": 0.46875,
+      "rewards/rejected": 2.125,
+      "step": 52
+    },
+    {
+      "epoch": 0.212,
+      "grad_norm": 13.359303723068443,
+      "learning_rate": 9.170821798439683e-07,
+      "logits/chosen": -0.921875,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -7.21875,
+      "logps/rejected": -19.75,
+      "loss": 1.4738,
+      "nll_loss": 0.7265625,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.25,
+      "rewards/margins": 0.84375,
+      "rewards/rejected": 1.4140625,
+      "step": 53
+    },
+    {
+      "epoch": 0.216,
+      "grad_norm": 8.607311027896719,
+      "learning_rate": 9.134673047280643e-07,
+      "logits/chosen": -0.90234375,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -7.3125,
+      "logps/rejected": -12.75,
+      "loss": 1.5091,
+      "nll_loss": 0.6875,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.53125,
+      "rewards/margins": 0.66796875,
+      "rewards/rejected": 1.859375,
+      "step": 54
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 10.84727026791628,
+      "learning_rate": 9.097827502627136e-07,
+      "logits/chosen": -0.890625,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -5.40625,
+      "logps/rejected": -15.125,
+      "loss": 1.3565,
+      "nll_loss": 0.50390625,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 2.90625,
+      "rewards/margins": 0.72265625,
+      "rewards/rejected": 2.1875,
+      "step": 55
+    },
+    {
+      "epoch": 0.224,
+      "grad_norm": 10.663731973556606,
+      "learning_rate": 9.06029137385425e-07,
+      "logits/chosen": -0.90234375,
+      "logits/rejected": -0.875,
+      "logps/chosen": -8.0625,
+      "logps/rejected": -16.875,
+      "loss": 1.4568,
+      "nll_loss": 0.77734375,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.890625,
+      "rewards/margins": 0.625,
+      "rewards/rejected": 2.265625,
+      "step": 56
+    },
+    {
+      "epoch": 0.228,
+      "grad_norm": 9.34103541373727,
+      "learning_rate": 9.022070986717378e-07,
+      "logits/chosen": -0.8984375,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -7.6875,
+      "logps/rejected": -20.75,
+      "loss": 1.4427,
+      "nll_loss": 0.71875,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.40625,
+      "rewards/margins": 1.125,
+      "rewards/rejected": 1.28125,
+      "step": 57
+    },
+    {
+      "epoch": 0.232,
+      "grad_norm": 9.905364057698737,
+      "learning_rate": 8.983172782286179e-07,
+      "logits/chosen": -0.9140625,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -6.46875,
+      "logps/rejected": -14.125,
+      "loss": 1.4267,
+      "nll_loss": 0.69140625,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.875,
+      "rewards/margins": 0.326171875,
+      "rewards/rejected": 2.546875,
+      "step": 58
+    },
+    {
+      "epoch": 0.236,
+      "grad_norm": 9.50298929918346,
+      "learning_rate": 8.943603315859101e-07,
+      "logits/chosen": -0.91796875,
+      "logits/rejected": -0.92578125,
+      "logps/chosen": -6.90625,
+      "logps/rejected": -16.25,
+      "loss": 1.3527,
+      "nll_loss": 0.64453125,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 2.921875,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": 1.484375,
+      "step": 59
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 9.418798977945164,
+      "learning_rate": 8.90336925585864e-07,
+      "logits/chosen": -0.9140625,
+      "logits/rejected": -0.94140625,
+      "logps/chosen": -5.25,
+      "logps/rejected": -19.0,
+      "loss": 1.4071,
+      "nll_loss": 0.546875,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 2.578125,
+      "rewards/margins": 1.03125,
+      "rewards/rejected": 1.5546875,
+      "step": 60
+    },
+    {
+      "epoch": 0.244,
+      "grad_norm": 12.471943357949753,
+      "learning_rate": 8.862477382707568e-07,
+      "logits/chosen": -0.9453125,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -7.8125,
+      "logps/rejected": -19.5,
+      "loss": 1.516,
+      "nll_loss": 0.6640625,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.640625,
+      "rewards/margins": 0.98828125,
+      "rewards/rejected": 1.6640625,
+      "step": 61
+    },
+    {
+      "epoch": 0.248,
+      "grad_norm": 10.996603171043574,
+      "learning_rate": 8.820934587686245e-07,
+      "logits/chosen": -0.94921875,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -7.21875,
+      "logps/rejected": -17.0,
+      "loss": 1.4125,
+      "nll_loss": 0.703125,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 2.40625,
+      "rewards/margins": 0.515625,
+      "rewards/rejected": 1.890625,
+      "step": 62
+    },
+    {
+      "epoch": 0.252,
+      "grad_norm": 10.253437222104349,
+      "learning_rate": 8.778747871771291e-07,
+      "logits/chosen": -0.91015625,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -5.75,
+      "logps/rejected": -18.75,
+      "loss": 1.3891,
+      "nll_loss": 0.55078125,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 2.9375,
+      "rewards/margins": 1.140625,
+      "rewards/rejected": 1.796875,
+      "step": 63
+    },
+    {
+      "epoch": 0.256,
+      "grad_norm": 9.86186104574288,
+      "learning_rate": 8.735924344455732e-07,
+      "logits/chosen": -0.90234375,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -6.875,
+      "logps/rejected": -18.5,
+      "loss": 1.3061,
+      "nll_loss": 0.67578125,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 3.015625,
+      "rewards/margins": 1.171875,
+      "rewards/rejected": 1.84375,
+      "step": 64
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 11.007376728694611,
+      "learning_rate": 8.692471222550885e-07,
+      "logits/chosen": -0.9453125,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -7.625,
+      "logps/rejected": -23.125,
+      "loss": 1.4576,
+      "nll_loss": 0.71875,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.625,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": 1.1484375,
+      "step": 65
+    },
+    {
+      "epoch": 0.264,
+      "grad_norm": 10.331552283839107,
+      "learning_rate": 8.64839582897015e-07,
+      "logits/chosen": -0.9375,
+      "logits/rejected": -0.953125,
+      "logps/chosen": -7.3125,
+      "logps/rejected": -13.5625,
+      "loss": 1.3471,
+      "nll_loss": 0.6953125,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 2.921875,
+      "rewards/margins": 0.384765625,
+      "rewards/rejected": 2.546875,
+      "step": 66
+    },
+    {
+      "epoch": 0.268,
+      "grad_norm": 9.444511843257002,
+      "learning_rate": 8.603705591494916e-07,
+      "logits/chosen": -0.91015625,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -6.53125,
+      "logps/rejected": -16.5,
+      "loss": 1.2306,
+      "nll_loss": 0.6796875,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.25,
+      "rewards/margins": 0.80078125,
+      "rewards/rejected": 1.4453125,
+      "step": 67
+    },
+    {
+      "epoch": 0.272,
+      "grad_norm": 12.577093117314861,
+      "learning_rate": 8.5584080415228e-07,
+      "logits/chosen": -0.9140625,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -5.46875,
+      "logps/rejected": -15.5625,
+      "loss": 1.2947,
+      "nll_loss": 0.5078125,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 2.65625,
+      "rewards/margins": 0.87890625,
+      "rewards/rejected": 1.78125,
+      "step": 68
+    },
+    {
+      "epoch": 0.276,
+      "grad_norm": 10.641105295546792,
+      "learning_rate": 8.512510812798425e-07,
+      "logits/chosen": -0.94140625,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -6.3125,
+      "logps/rejected": -16.875,
+      "loss": 1.2781,
+      "nll_loss": 0.578125,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 2.921875,
+      "rewards/margins": 1.0546875,
+      "rewards/rejected": 1.875,
+      "step": 69
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 12.570856113406187,
+      "learning_rate": 8.466021640126945e-07,
+      "logits/chosen": -0.953125,
+      "logits/rejected": -0.984375,
+      "logps/chosen": -5.34375,
+      "logps/rejected": -18.625,
+      "loss": 1.3463,
+      "nll_loss": 0.490234375,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.9375,
+      "rewards/margins": 1.25,
+      "rewards/rejected": 1.6953125,
+      "step": 70
+    },
+    {
+      "epoch": 0.284,
+      "grad_norm": 12.634804813000724,
+      "learning_rate": 8.418948358070535e-07,
+      "logits/chosen": -0.9453125,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -7.125,
+      "logps/rejected": -17.25,
+      "loss": 1.2968,
+      "nll_loss": 0.66015625,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 2.703125,
+      "rewards/margins": 0.9609375,
+      "rewards/rejected": 1.7421875,
+      "step": 71
+    },
+    {
+      "epoch": 0.288,
+      "grad_norm": 11.739576042187055,
+      "learning_rate": 8.37129889962809e-07,
+      "logits/chosen": -1.0,
+      "logits/rejected": -0.9765625,
+      "logps/chosen": -7.4375,
+      "logps/rejected": -20.875,
+      "loss": 1.3581,
+      "nll_loss": 0.67578125,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.703125,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": 1.40625,
+      "step": 72
+    },
+    {
+      "epoch": 0.292,
+      "grad_norm": 12.097928995634907,
+      "learning_rate": 8.323081294898307e-07,
+      "logits/chosen": -0.953125,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -5.3125,
+      "logps/rejected": -21.5,
+      "loss": 1.199,
+      "nll_loss": 0.56640625,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.75,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": 1.4609375,
+      "step": 73
+    },
+    {
+      "epoch": 0.296,
+      "grad_norm": 10.920824389039185,
+      "learning_rate": 8.274303669726426e-07,
+      "logits/chosen": -0.95703125,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -5.75,
+      "logps/rejected": -18.625,
+      "loss": 1.3358,
+      "nll_loss": 0.51953125,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 2.609375,
+      "rewards/margins": 1.15625,
+      "rewards/rejected": 1.4609375,
+      "step": 74
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 14.177002319797584,
+      "learning_rate": 8.22497424433482e-07,
+      "logits/chosen": -0.93359375,
+      "logits/rejected": -0.9296875,
+      "logps/chosen": -6.125,
+      "logps/rejected": -17.25,
+      "loss": 1.2862,
+      "nll_loss": 0.61328125,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.890625,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": 1.5,
+      "step": 75
+    },
+    {
+      "epoch": 0.304,
+      "grad_norm": 13.6699848015144,
+      "learning_rate": 8.175101331937692e-07,
+      "logits/chosen": -0.94921875,
+      "logits/rejected": -0.95703125,
+      "logps/chosen": -6.34375,
+      "logps/rejected": -15.625,
+      "loss": 1.2559,
+      "nll_loss": 0.66015625,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 2.8125,
+      "rewards/margins": 1.140625,
+      "rewards/rejected": 1.671875,
+      "step": 76
+    },
+    {
+      "epoch": 0.308,
+      "grad_norm": 13.010408259816026,
+      "learning_rate": 8.124693337340092e-07,
+      "logits/chosen": -0.99609375,
+      "logits/rejected": -0.96484375,
+      "logps/chosen": -6.96875,
+      "logps/rejected": -21.625,
+      "loss": 1.3234,
+      "nll_loss": 0.6875,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.890625,
+      "rewards/margins": 1.3125,
+      "rewards/rejected": 1.5703125,
+      "step": 77
+    },
+    {
+      "epoch": 0.312,
+      "grad_norm": 14.267160801384883,
+      "learning_rate": 8.073758755521504e-07,
+      "logits/chosen": -0.99609375,
+      "logits/rejected": -0.98828125,
+      "logps/chosen": -6.1875,
+      "logps/rejected": -23.875,
+      "loss": 1.2073,
+      "nll_loss": 0.6171875,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 2.921875,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": 1.09375,
+      "step": 78
+    },
+    {
+      "epoch": 0.316,
+      "grad_norm": 10.978973661742327,
+      "learning_rate": 8.022306170204231e-07,
+      "logits/chosen": -1.0234375,
+      "logits/rejected": -1.0,
+      "logps/chosen": -5.8125,
+      "logps/rejected": -23.5,
+      "loss": 1.3188,
+      "nll_loss": 0.5625,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 3.09375,
+      "rewards/margins": 2.046875,
+      "rewards/rejected": 1.0390625,
+      "step": 79
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 11.702933301921057,
+      "learning_rate": 7.970344252406831e-07,
+      "logits/chosen": -0.9765625,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -5.03125,
+      "logps/rejected": -18.5,
+      "loss": 1.3078,
+      "nll_loss": 0.48828125,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 2.90625,
+      "rewards/margins": 1.2109375,
+      "rewards/rejected": 1.6875,
+      "step": 80
+    },
+    {
+      "epoch": 0.324,
+      "grad_norm": 12.12062750517865,
+      "learning_rate": 7.917881758982836e-07,
+      "logits/chosen": -0.953125,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -6.0,
+      "logps/rejected": -15.875,
+      "loss": 1.2015,
+      "nll_loss": 0.59765625,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.65625,
+      "rewards/margins": 0.796875,
+      "rewards/rejected": 1.859375,
+      "step": 81
+    },
+    {
+      "epoch": 0.328,
+      "grad_norm": 14.451380371080663,
+      "learning_rate": 7.86492753114501e-07,
+      "logits/chosen": -1.015625,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -6.03125,
+      "logps/rejected": -24.875,
+      "loss": 1.173,
+      "nll_loss": 0.55859375,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 2.859375,
+      "rewards/margins": 2.078125,
+      "rewards/rejected": 0.7734375,
+      "step": 82
+    },
+    {
+      "epoch": 0.332,
+      "grad_norm": 12.677944938939618,
+      "learning_rate": 7.811490492975389e-07,
+      "logits/chosen": -0.9609375,
+      "logits/rejected": -0.953125,
+      "logps/chosen": -4.5,
+      "logps/rejected": -20.5,
+      "loss": 1.2099,
+      "nll_loss": 0.478515625,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.703125,
+      "rewards/margins": 1.1484375,
+      "rewards/rejected": 1.5625,
+      "step": 83
+    },
+    {
+      "epoch": 0.336,
+      "grad_norm": 12.912178140489386,
+      "learning_rate": 7.757579649921353e-07,
+      "logits/chosen": -1.0234375,
+      "logits/rejected": -0.99609375,
+      "logps/chosen": -4.96875,
+      "logps/rejected": -26.0,
+      "loss": 1.2007,
+      "nll_loss": 0.48828125,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 2.796875,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": 1.328125,
+      "step": 84
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 11.95102680724988,
+      "learning_rate": 7.703204087277988e-07,
+      "logits/chosen": -1.0,
+      "logits/rejected": -0.9921875,
+      "logps/chosen": -5.125,
+      "logps/rejected": -20.75,
+      "loss": 1.1374,
+      "nll_loss": 0.470703125,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.640625,
+      "rewards/margins": 1.1953125,
+      "rewards/rejected": 1.4453125,
+      "step": 85
+    },
+    {
+      "epoch": 0.344,
+      "grad_norm": 15.312482138498904,
+      "learning_rate": 7.648372968656993e-07,
+      "logits/chosen": -0.9921875,
+      "logits/rejected": -0.99609375,
+      "logps/chosen": -6.15625,
+      "logps/rejected": -30.5,
+      "loss": 1.1599,
+      "nll_loss": 0.55859375,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.515625,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": 0.75,
+      "step": 86
+    },
+    {
+      "epoch": 0.348,
+      "grad_norm": 13.90159120685156,
+      "learning_rate": 7.593095534442386e-07,
+      "logits/chosen": -0.9921875,
+      "logits/rejected": -0.984375,
+      "logps/chosen": -4.84375,
+      "logps/rejected": -19.375,
+      "loss": 1.116,
+      "nll_loss": 0.52734375,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.9375,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": 1.46875,
+      "step": 87
+    },
+    {
+      "epoch": 0.352,
+      "grad_norm": 14.950945255344106,
+      "learning_rate": 7.537381100233278e-07,
+      "logits/chosen": -1.0078125,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -5.0625,
+      "logps/rejected": -23.375,
+      "loss": 1.2363,
+      "nll_loss": 0.482421875,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.640625,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": 0.9609375,
+      "step": 88
+    },
+    {
+      "epoch": 0.356,
+      "grad_norm": 12.117659621877523,
+      "learning_rate": 7.481239055273958e-07,
+      "logits/chosen": -0.94921875,
+      "logits/rejected": -0.9609375,
+      "logps/chosen": -4.625,
+      "logps/rejected": -12.8125,
+      "loss": 1.169,
+      "nll_loss": 0.462890625,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.828125,
+      "rewards/margins": 0.546875,
+      "rewards/rejected": 2.28125,
+      "step": 89
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 15.946889828294005,
+      "learning_rate": 7.424678860871584e-07,
+      "logits/chosen": -1.0078125,
+      "logits/rejected": -0.984375,
+      "logps/chosen": -6.125,
+      "logps/rejected": -28.0,
+      "loss": 1.185,
+      "nll_loss": 0.58203125,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 2.5625,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": 0.8515625,
+      "step": 90
+    },
+    {
+      "epoch": 0.364,
+      "grad_norm": 21.734506154061158,
+      "learning_rate": 7.367710048801715e-07,
+      "logits/chosen": -1.015625,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -3.390625,
+      "logps/rejected": -19.375,
+      "loss": 1.1948,
+      "nll_loss": 0.337890625,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.703125,
+      "rewards/margins": 1.2421875,
+      "rewards/rejected": 1.453125,
+      "step": 91
+    },
+    {
+      "epoch": 0.368,
+      "grad_norm": 12.733439669594082,
+      "learning_rate": 7.31034221970198e-07,
+      "logits/chosen": -1.03125,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -5.3125,
+      "logps/rejected": -29.625,
+      "loss": 1.1723,
+      "nll_loss": 0.55078125,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 2.828125,
+      "rewards/margins": 2.015625,
+      "rewards/rejected": 0.81640625,
+      "step": 92
+    },
+    {
+      "epoch": 0.372,
+      "grad_norm": 13.39954488299055,
+      "learning_rate": 7.252585041454133e-07,
+      "logits/chosen": -1.015625,
+      "logits/rejected": -1.0,
+      "logps/chosen": -4.4375,
+      "logps/rejected": -29.5,
+      "loss": 1.1454,
+      "nll_loss": 0.443359375,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 2.6875,
+      "rewards/margins": 2.15625,
+      "rewards/rejected": 0.5390625,
+      "step": 93
+    },
+    {
+      "epoch": 0.376,
+      "grad_norm": 15.64435560107329,
+      "learning_rate": 7.194448247554778e-07,
+      "logits/chosen": -1.0,
+      "logits/rejected": -0.9921875,
+      "logps/chosen": -5.15625,
+      "logps/rejected": -20.0,
+      "loss": 1.1604,
+      "nll_loss": 0.546875,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.8125,
+      "rewards/margins": 1.1015625,
+      "rewards/rejected": 1.703125,
+      "step": 94
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 17.13096115933346,
+      "learning_rate": 7.135941635475036e-07,
+      "logits/chosen": -1.015625,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -6.8125,
+      "logps/rejected": -28.5,
+      "loss": 1.2684,
+      "nll_loss": 0.6015625,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.953125,
+      "rewards/margins": 1.9921875,
+      "rewards/rejected": 0.96875,
+      "step": 95
+    },
+    {
+      "epoch": 0.384,
+      "grad_norm": 16.845492514129948,
+      "learning_rate": 7.077075065009433e-07,
+      "logits/chosen": -1.0234375,
+      "logits/rejected": -0.99609375,
+      "logps/chosen": -6.0625,
+      "logps/rejected": -22.625,
+      "loss": 1.2163,
+      "nll_loss": 0.5859375,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.3125,
+      "rewards/margins": 1.2265625,
+      "rewards/rejected": 1.0859375,
+      "step": 96
+    },
+    {
+      "epoch": 0.388,
+      "grad_norm": 24.19829824906313,
+      "learning_rate": 7.017858456614283e-07,
+      "logits/chosen": -1.03125,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -4.8125,
+      "logps/rejected": -24.125,
+      "loss": 1.2011,
+      "nll_loss": 0.46875,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 3.0,
+      "rewards/margins": 1.75,
+      "rewards/rejected": 1.2578125,
+      "step": 97
+    },
+    {
+      "epoch": 0.392,
+      "grad_norm": 16.528677943117593,
+      "learning_rate": 6.958301789735852e-07,
+      "logits/chosen": -1.03125,
+      "logits/rejected": -1.0234375,
+      "logps/chosen": -5.46875,
+      "logps/rejected": -26.125,
+      "loss": 1.2671,
+      "nll_loss": 0.5546875,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.5,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": 0.66015625,
+      "step": 98
+    },
+    {
+      "epoch": 0.396,
+      "grad_norm": 15.542683127815902,
+      "learning_rate": 6.898415101128569e-07,
+      "logits/chosen": -1.0234375,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -5.5625,
+      "logps/rejected": -18.25,
+      "loss": 1.2051,
+      "nll_loss": 0.53125,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.796875,
+      "rewards/margins": 1.109375,
+      "rewards/rejected": 1.6796875,
+      "step": 99
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 16.703597419369395,
+      "learning_rate": 6.8382084831636e-07,
+      "logits/chosen": -1.0546875,
+      "logits/rejected": -1.0234375,
+      "logps/chosen": -4.34375,
+      "logps/rejected": -30.875,
+      "loss": 1.1695,
+      "nll_loss": 0.392578125,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 2.46875,
+      "rewards/margins": 2.140625,
+      "rewards/rejected": 0.333984375,
+      "step": 100
+    },
+    {
+      "epoch": 0.404,
+      "grad_norm": 20.289662822876423,
+      "learning_rate": 6.777692082128024e-07,
+      "logits/chosen": -1.046875,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -6.9375,
+      "logps/rejected": -23.125,
+      "loss": 1.1355,
+      "nll_loss": 0.6328125,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 2.84375,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": 1.109375,
+      "step": 101
+    },
+    {
+      "epoch": 0.408,
+      "grad_norm": 10.57908068816297,
+      "learning_rate": 6.716876096514943e-07,
+      "logits/chosen": -1.0078125,
+      "logits/rejected": -1.0234375,
+      "logps/chosen": -3.34375,
+      "logps/rejected": -35.25,
+      "loss": 1.1203,
+      "nll_loss": 0.30078125,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.75,
+      "rewards/margins": 2.953125,
+      "rewards/rejected": -0.2001953125,
+      "step": 102
+    },
+    {
+      "epoch": 0.412,
+      "grad_norm": 12.448311996517877,
+      "learning_rate": 6.655770775304782e-07,
+      "logits/chosen": -0.98828125,
+      "logits/rejected": -0.98828125,
+      "logps/chosen": -5.5,
+      "logps/rejected": -13.8125,
+      "loss": 1.2512,
+      "nll_loss": 0.5625,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 2.90625,
+      "rewards/margins": 0.66015625,
+      "rewards/rejected": 2.25,
+      "step": 103
+    },
+    {
+      "epoch": 0.416,
+      "grad_norm": 19.442937641890275,
+      "learning_rate": 6.594386416238094e-07,
+      "logits/chosen": -0.98046875,
+      "logits/rejected": -0.98828125,
+      "logps/chosen": -8.0625,
+      "logps/rejected": -20.0,
+      "loss": 1.1029,
+      "nll_loss": 0.75,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 2.546875,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": 1.1484375,
+      "step": 104
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 17.597984347295238,
+      "learning_rate": 6.532733364080125e-07,
+      "logits/chosen": -0.98828125,
+      "logits/rejected": -0.97265625,
+      "logps/chosen": -5.53125,
+      "logps/rejected": -26.125,
+      "loss": 1.0465,
+      "nll_loss": 0.5078125,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.78125,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": 0.9375,
+      "step": 105
+    },
+    {
+      "epoch": 0.424,
+      "grad_norm": 42.65485477575747,
+      "learning_rate": 6.470822008877482e-07,
+      "logits/chosen": -1.0,
+      "logits/rejected": -0.98828125,
+      "logps/chosen": -5.9375,
+      "logps/rejected": -19.125,
+      "loss": 1.1754,
+      "nll_loss": 0.53515625,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.875,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": 1.4921875,
+      "step": 106
+    },
+    {
+      "epoch": 0.428,
+      "grad_norm": 14.418794180437754,
+      "learning_rate": 6.408662784207149e-07,
+      "logits/chosen": -1.0078125,
+      "logits/rejected": -1.0,
+      "logps/chosen": -4.875,
+      "logps/rejected": -28.375,
+      "loss": 1.1556,
+      "nll_loss": 0.51171875,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.71875,
+      "rewards/margins": 1.875,
+      "rewards/rejected": 0.84375,
+      "step": 107
+    },
+    {
+      "epoch": 0.432,
+      "grad_norm": 24.069184880501325,
+      "learning_rate": 6.346266165418172e-07,
+      "logits/chosen": -0.9765625,
+      "logits/rejected": -0.95703125,
+      "logps/chosen": -4.71875,
+      "logps/rejected": -23.0,
+      "loss": 1.1594,
+      "nll_loss": 0.48046875,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 3.15625,
+      "rewards/margins": 1.875,
+      "rewards/rejected": 1.2734375,
+      "step": 108
+    },
+    {
+      "epoch": 0.436,
+      "grad_norm": 33.99711892618498,
+      "learning_rate": 6.283642667866315e-07,
+      "logits/chosen": -1.03125,
+      "logits/rejected": -0.98046875,
+      "logps/chosen": -6.28125,
+      "logps/rejected": -27.375,
+      "loss": 1.1293,
+      "nll_loss": 0.58984375,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.5625,
+      "rewards/margins": 1.625,
+      "rewards/rejected": 0.9375,
+      "step": 109
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 35.15701910686572,
+      "learning_rate": 6.220802845141957e-07,
+      "logits/chosen": -1.03125,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -5.71875,
+      "logps/rejected": -37.25,
+      "loss": 1.1224,
+      "nll_loss": 0.51953125,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.578125,
+      "rewards/margins": 2.609375,
+      "rewards/rejected": -0.03564453125,
+      "step": 110
+    },
+    {
+      "epoch": 0.444,
+      "grad_norm": 16.617005010321233,
+      "learning_rate": 6.157757287291556e-07,
+      "logits/chosen": -1.015625,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -3.859375,
+      "logps/rejected": -31.75,
+      "loss": 1.0481,
+      "nll_loss": 0.40234375,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 2.6875,
+      "rewards/margins": 2.6875,
+      "rewards/rejected": 0.0074462890625,
+      "step": 111
+    },
+    {
+      "epoch": 0.448,
+      "grad_norm": 12.591607226736722,
+      "learning_rate": 6.094516619032975e-07,
+      "logits/chosen": -1.046875,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -5.125,
+      "logps/rejected": -26.5,
+      "loss": 1.0584,
+      "nll_loss": 0.51171875,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.71875,
+      "rewards/margins": 1.90625,
+      "rewards/rejected": 0.82421875,
+      "step": 112
+    },
+    {
+      "epoch": 0.452,
+      "grad_norm": 12.509284400507388,
+      "learning_rate": 6.03109149796494e-07,
+      "logits/chosen": -1.0546875,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -3.984375,
+      "logps/rejected": -23.0,
+      "loss": 1.0653,
+      "nll_loss": 0.390625,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 2.859375,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": 1.0625,
+      "step": 113
+    },
+    {
+      "epoch": 0.456,
+      "grad_norm": 33.81401585194403,
+      "learning_rate": 5.967492612770997e-07,
+      "logits/chosen": -1.03125,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -6.65625,
+      "logps/rejected": -28.125,
+      "loss": 1.0973,
+      "nll_loss": 0.6640625,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.625,
+      "rewards/margins": 2.015625,
+      "rewards/rejected": 0.6015625,
+      "step": 114
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 15.815532311957057,
+      "learning_rate": 5.90373068141819e-07,
+      "logits/chosen": -1.0,
+      "logits/rejected": -0.9765625,
+      "logps/chosen": -6.875,
+      "logps/rejected": -29.125,
+      "loss": 1.2058,
+      "nll_loss": 0.69140625,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.390625,
+      "rewards/margins": 2.015625,
+      "rewards/rejected": 0.375,
+      "step": 115
+    },
+    {
+      "epoch": 0.464,
+      "grad_norm": 22.102704341248568,
+      "learning_rate": 5.839816449350824e-07,
+      "logits/chosen": -0.9921875,
+      "logits/rejected": -0.9921875,
+      "logps/chosen": -5.78125,
+      "logps/rejected": -23.625,
+      "loss": 1.1099,
+      "nll_loss": 0.5703125,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 3.015625,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": 1.46875,
+      "step": 116
+    },
+    {
+      "epoch": 0.468,
+      "grad_norm": 16.24899670304244,
+      "learning_rate": 5.775760687679603e-07,
+      "logits/chosen": -1.0078125,
+      "logits/rejected": -0.984375,
+      "logps/chosen": -4.28125,
+      "logps/rejected": -23.125,
+      "loss": 1.0259,
+      "nll_loss": 0.41015625,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.78125,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": 1.0234375,
+      "step": 117
+    },
+    {
+      "epoch": 0.472,
+      "grad_norm": 14.705074940426503,
+      "learning_rate": 5.711574191366427e-07,
+      "logits/chosen": -1.0234375,
+      "logits/rejected": -1.0234375,
+      "logps/chosen": -3.203125,
+      "logps/rejected": -25.875,
+      "loss": 1.1279,
+      "nll_loss": 0.330078125,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 3.0,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": 1.1953125,
+      "step": 118
+    },
+    {
+      "epoch": 0.476,
+      "grad_norm": 20.221686205664327,
+      "learning_rate": 5.647267777405177e-07,
+      "logits/chosen": -1.0234375,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -4.40625,
+      "logps/rejected": -23.375,
+      "loss": 1.1415,
+      "nll_loss": 0.478515625,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.578125,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": 1.1796875,
+      "step": 119
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 24.368984032622865,
+      "learning_rate": 5.582852282998795e-07,
+      "logits/chosen": -1.0234375,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -5.40625,
+      "logps/rejected": -23.125,
+      "loss": 1.0667,
+      "nll_loss": 0.474609375,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.84375,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": 1.296875,
+      "step": 120
+    },
+    {
+      "epoch": 0.484,
+      "grad_norm": 22.432812053652427,
+      "learning_rate": 5.518338563732944e-07,
+      "logits/chosen": -1.046875,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -4.90625,
+      "logps/rejected": -23.625,
+      "loss": 1.0827,
+      "nll_loss": 0.54296875,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.546875,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": 0.90625,
+      "step": 121
+    },
+    {
+      "epoch": 0.488,
+      "grad_norm": 16.17860106216787,
+      "learning_rate": 5.453737491746572e-07,
+      "logits/chosen": -1.0234375,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -3.625,
+      "logps/rejected": -22.875,
+      "loss": 1.0798,
+      "nll_loss": 0.375,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 3.046875,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": 1.4296875,
+      "step": 122
+    },
+    {
+      "epoch": 0.492,
+      "grad_norm": 24.70109578919281,
+      "learning_rate": 5.389059953899698e-07,
+      "logits/chosen": -1.09375,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -4.53125,
+      "logps/rejected": -31.375,
+      "loss": 1.2218,
+      "nll_loss": 0.451171875,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 3.109375,
+      "rewards/margins": 2.34375,
+      "rewards/rejected": 0.765625,
+      "step": 123
+    },
+    {
+      "epoch": 0.496,
+      "grad_norm": 31.00791322114362,
+      "learning_rate": 5.324316849938714e-07,
+      "logits/chosen": -0.98046875,
+      "logits/rejected": -0.97265625,
+      "logps/chosen": -6.375,
+      "logps/rejected": -21.75,
+      "loss": 1.166,
+      "nll_loss": 0.57421875,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.71875,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": 1.15625,
+      "step": 124
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 37.45373935605177,
+      "learning_rate": 5.259519090659498e-07,
+      "logits/chosen": -1.0546875,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -4.40625,
+      "logps/rejected": -25.5,
+      "loss": 1.0699,
+      "nll_loss": 0.392578125,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.890625,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": 1.1953125,
+      "step": 125
+    },
+    {
+      "epoch": 0.504,
+      "grad_norm": 12.712491511773225,
+      "learning_rate": 5.194677596068688e-07,
+      "logits/chosen": -1.0078125,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -5.84375,
+      "logps/rejected": -21.375,
+      "loss": 1.1111,
+      "nll_loss": 0.55859375,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.65625,
+      "rewards/margins": 1.1328125,
+      "rewards/rejected": 1.5234375,
+      "step": 126
+    },
+    {
+      "epoch": 0.508,
+      "grad_norm": 14.71397944686033,
+      "learning_rate": 5.129803293543391e-07,
+      "logits/chosen": -1.0546875,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -4.375,
+      "logps/rejected": -28.875,
+      "loss": 1.0949,
+      "nll_loss": 0.41015625,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.984375,
+      "rewards/margins": 2.390625,
+      "rewards/rejected": 0.58984375,
+      "step": 127
+    },
+    {
+      "epoch": 0.512,
+      "grad_norm": 20.40214155927554,
+      "learning_rate": 5.064907115989654e-07,
+      "logits/chosen": -1.078125,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -5.375,
+      "logps/rejected": -30.25,
+      "loss": 1.0997,
+      "nll_loss": 0.46484375,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.984375,
+      "rewards/margins": 2.84375,
+      "rewards/rejected": 0.1357421875,
+      "step": 128
+    },
+    {
+      "epoch": 0.516,
+      "grad_norm": 20.076566430152866,
+      "learning_rate": 5e-07,
+      "logits/chosen": -1.0546875,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -5.84375,
+      "logps/rejected": -31.5,
+      "loss": 1.1843,
+      "nll_loss": 0.5703125,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.484375,
+      "rewards/margins": 1.984375,
+      "rewards/rejected": 0.5078125,
+      "step": 129
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 15.69512072569519,
+      "learning_rate": 4.935092884010346e-07,
+      "logits/chosen": -1.0234375,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -4.6875,
+      "logps/rejected": -24.75,
+      "loss": 1.0591,
+      "nll_loss": 0.4609375,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.84375,
+      "rewards/margins": 1.8828125,
+      "rewards/rejected": 0.96484375,
+      "step": 130
+    },
+    {
+      "epoch": 0.524,
+      "grad_norm": 15.184658306188565,
+      "learning_rate": 4.870196706456609e-07,
+      "logits/chosen": -1.0625,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -5.90625,
+      "logps/rejected": -29.5,
+      "loss": 1.1098,
+      "nll_loss": 0.58203125,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 2.828125,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": 1.1171875,
+      "step": 131
+    },
+    {
+      "epoch": 0.528,
+      "grad_norm": 30.109778759207604,
+      "learning_rate": 4.805322403931311e-07,
+      "logits/chosen": -1.109375,
+      "logits/rejected": -1.1015625,
+      "logps/chosen": -4.78125,
+      "logps/rejected": -28.25,
+      "loss": 1.0583,
+      "nll_loss": 0.41796875,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 3.078125,
+      "rewards/margins": 2.765625,
+      "rewards/rejected": 0.30859375,
+      "step": 132
+    },
+    {
+      "epoch": 0.532,
+      "grad_norm": 19.180407490964914,
+      "learning_rate": 4.7404809093405017e-07,
+      "logits/chosen": -1.0703125,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -4.5625,
+      "logps/rejected": -24.5,
+      "loss": 1.0781,
+      "nll_loss": 0.3984375,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.6875,
+      "rewards/margins": 1.75,
+      "rewards/rejected": 0.9375,
+      "step": 133
+    },
+    {
+      "epoch": 0.536,
+      "grad_norm": 11.865585536243879,
+      "learning_rate": 4.675683150061284e-07,
+      "logits/chosen": -1.078125,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -4.65625,
+      "logps/rejected": -27.75,
+      "loss": 1.1264,
+      "nll_loss": 0.470703125,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 3.140625,
+      "rewards/margins": 1.9375,
+      "rewards/rejected": 1.203125,
+      "step": 134
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 14.216123496302846,
+      "learning_rate": 4.6109400461003003e-07,
+      "logits/chosen": -1.0234375,
+      "logits/rejected": -1.0234375,
+      "logps/chosen": -5.34375,
+      "logps/rejected": -23.5,
+      "loss": 1.0383,
+      "nll_loss": 0.453125,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 3.3125,
+      "rewards/margins": 2.25,
+      "rewards/rejected": 1.0703125,
+      "step": 135
+    },
+    {
+      "epoch": 0.544,
+      "grad_norm": 28.567567684765535,
+      "learning_rate": 4.5462625082534286e-07,
+      "logits/chosen": -1.078125,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -3.9375,
+      "logps/rejected": -28.125,
+      "loss": 0.9913,
+      "nll_loss": 0.388671875,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 2.875,
+      "rewards/margins": 2.53125,
+      "rewards/rejected": 0.353515625,
+      "step": 136
+    },
+    {
+      "epoch": 0.548,
+      "grad_norm": 19.382351209373788,
+      "learning_rate": 4.481661436267057e-07,
+      "logits/chosen": -1.0703125,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -4.65625,
+      "logps/rejected": -33.5,
+      "loss": 1.1359,
+      "nll_loss": 0.42578125,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.828125,
+      "rewards/margins": 2.328125,
+      "rewards/rejected": 0.5078125,
+      "step": 137
+    },
+    {
+      "epoch": 0.552,
+      "grad_norm": 28.26009731578296,
+      "learning_rate": 4.4171477170012043e-07,
+      "logits/chosen": -1.0546875,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -4.21875,
+      "logps/rejected": -32.0,
+      "loss": 1.1338,
+      "nll_loss": 0.380859375,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 2.953125,
+      "rewards/margins": 2.578125,
+      "rewards/rejected": 0.37890625,
+      "step": 138
+    },
+    {
+      "epoch": 0.556,
+      "grad_norm": 48.13783869940889,
+      "learning_rate": 4.352732222594823e-07,
+      "logits/chosen": -1.046875,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -5.1875,
+      "logps/rejected": -25.625,
+      "loss": 1.1776,
+      "nll_loss": 0.466796875,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 2.859375,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": 1.0078125,
+      "step": 139
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 23.304268399995618,
+      "learning_rate": 4.2884258086335745e-07,
+      "logits/chosen": -1.0390625,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -5.65625,
+      "logps/rejected": -18.5,
+      "loss": 1.1203,
+      "nll_loss": 0.490234375,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 3.03125,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": 1.6015625,
+      "step": 140
+    },
+    {
+      "epoch": 0.564,
+      "grad_norm": 19.29343324391366,
+      "learning_rate": 4.224239312320398e-07,
+      "logits/chosen": -1.078125,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -7.90625,
+      "logps/rejected": -29.375,
+      "loss": 1.1849,
+      "nll_loss": 0.67578125,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.5,
+      "rewards/margins": 1.9375,
+      "rewards/rejected": 0.5625,
+      "step": 141
+    },
+    {
+      "epoch": 0.568,
+      "grad_norm": 17.35581169203161,
+      "learning_rate": 4.160183550649175e-07,
+      "logits/chosen": -1.0546875,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -4.65625,
+      "logps/rejected": -28.25,
+      "loss": 1.0433,
+      "nll_loss": 0.48046875,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 2.796875,
+      "rewards/margins": 2.03125,
+      "rewards/rejected": 0.76953125,
+      "step": 142
+    },
+    {
+      "epoch": 0.572,
+      "grad_norm": 31.43926475473609,
+      "learning_rate": 4.09626931858181e-07,
+      "logits/chosen": -1.0625,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -4.21875,
+      "logps/rejected": -19.125,
+      "loss": 1.112,
+      "nll_loss": 0.41015625,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.96875,
+      "rewards/margins": 1.3515625,
+      "rewards/rejected": 1.625,
+      "step": 143
+    },
+    {
+      "epoch": 0.576,
+      "grad_norm": 19.985361598611938,
+      "learning_rate": 4.0325073872290015e-07,
+      "logits/chosen": -1.0234375,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -6.6875,
+      "logps/rejected": -26.375,
+      "loss": 1.1101,
+      "nll_loss": 0.5859375,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.859375,
+      "rewards/margins": 2.328125,
+      "rewards/rejected": 0.52734375,
+      "step": 144
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 35.012979635813075,
+      "learning_rate": 3.96890850203506e-07,
+      "logits/chosen": -1.0625,
+      "logits/rejected": -1.0625,
+      "logps/chosen": -6.375,
+      "logps/rejected": -26.625,
+      "loss": 1.0092,
+      "nll_loss": 0.59765625,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 3.21875,
+      "rewards/margins": 2.265625,
+      "rewards/rejected": 0.94921875,
+      "step": 145
+    },
+    {
+      "epoch": 0.584,
+      "grad_norm": 20.26642224735176,
+      "learning_rate": 3.9054833809670265e-07,
+      "logits/chosen": -1.0546875,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -4.625,
+      "logps/rejected": -29.5,
+      "loss": 1.0404,
+      "nll_loss": 0.4140625,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.75,
+      "rewards/margins": 2.5,
+      "rewards/rejected": 0.2392578125,
+      "step": 146
+    },
+    {
+      "epoch": 0.588,
+      "grad_norm": 13.154350173457996,
+      "learning_rate": 3.8422427127084436e-07,
+      "logits/chosen": -1.078125,
+      "logits/rejected": -1.0625,
+      "logps/chosen": -6.53125,
+      "logps/rejected": -31.875,
+      "loss": 0.9968,
+      "nll_loss": 0.58203125,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.921875,
+      "rewards/margins": 2.59375,
+      "rewards/rejected": 0.32421875,
+      "step": 147
+    },
+    {
+      "epoch": 0.592,
+      "grad_norm": 23.129629439239768,
+      "learning_rate": 3.779197154858044e-07,
+      "logits/chosen": -1.0546875,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -5.21875,
+      "logps/rejected": -23.375,
+      "loss": 0.9492,
+      "nll_loss": 0.4921875,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 3.09375,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": 1.3671875,
+      "step": 148
+    },
+    {
+      "epoch": 0.596,
+      "grad_norm": 14.081159200332113,
+      "learning_rate": 3.716357332133686e-07,
+      "logits/chosen": -1.1171875,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -5.65625,
+      "logps/rejected": -30.5,
+      "loss": 1.0858,
+      "nll_loss": 0.58984375,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.828125,
+      "rewards/margins": 2.0625,
+      "rewards/rejected": 0.76953125,
+      "step": 149
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 30.59966116072142,
+      "learning_rate": 3.653733834581827e-07,
+      "logits/chosen": -1.09375,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -4.0,
+      "logps/rejected": -26.125,
+      "loss": 1.0947,
+      "nll_loss": 0.373046875,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.765625,
+      "rewards/margins": 2.21875,
+      "rewards/rejected": 0.546875,
+      "step": 150
+    },
+    {
+      "epoch": 0.604,
+      "grad_norm": 14.36839336730757,
+      "learning_rate": 3.591337215792851e-07,
+      "logits/chosen": -1.09375,
+      "logits/rejected": -1.109375,
+      "logps/chosen": -4.5625,
+      "logps/rejected": -27.5,
+      "loss": 1.085,
+      "nll_loss": 0.45703125,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.90625,
+      "rewards/margins": 1.8984375,
+      "rewards/rejected": 1.0,
+      "step": 151
+    },
+    {
+      "epoch": 0.608,
+      "grad_norm": 25.95175246611989,
+      "learning_rate": 3.5291779911225185e-07,
+      "logits/chosen": -1.0859375,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -4.5,
+      "logps/rejected": -30.75,
+      "loss": 1.0704,
+      "nll_loss": 0.427734375,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 2.984375,
+      "rewards/margins": 2.265625,
+      "rewards/rejected": 0.71484375,
+      "step": 152
+    },
+    {
+      "epoch": 0.612,
+      "grad_norm": 30.341569831825527,
+      "learning_rate": 3.4672666359198757e-07,
+      "logits/chosen": -1.078125,
+      "logits/rejected": -1.0625,
+      "logps/chosen": -4.21875,
+      "logps/rejected": -27.75,
+      "loss": 1.0871,
+      "nll_loss": 0.453125,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 3.265625,
+      "rewards/margins": 2.359375,
+      "rewards/rejected": 0.91015625,
+      "step": 153
+    },
+    {
+      "epoch": 0.616,
+      "grad_norm": 15.724055467705009,
+      "learning_rate": 3.405613583761907e-07,
+      "logits/chosen": -1.0546875,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -5.53125,
+      "logps/rejected": -22.25,
+      "loss": 1.0548,
+      "nll_loss": 0.5625,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 3.078125,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": 1.5234375,
+      "step": 154
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 23.186267615438663,
+      "learning_rate": 3.3442292246952186e-07,
+      "logits/chosen": -1.109375,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -5.71875,
+      "logps/rejected": -33.75,
+      "loss": 1.0691,
+      "nll_loss": 0.5390625,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.5,
+      "rewards/margins": 2.390625,
+      "rewards/rejected": 0.10595703125,
+      "step": 155
+    },
+    {
+      "epoch": 0.624,
+      "grad_norm": 19.08999867823956,
+      "learning_rate": 3.283123903485059e-07,
+      "logits/chosen": -1.0546875,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -6.15625,
+      "logps/rejected": -28.625,
+      "loss": 1.1063,
+      "nll_loss": 0.6015625,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 3.046875,
+      "rewards/margins": 2.03125,
+      "rewards/rejected": 1.015625,
+      "step": 156
+    },
+    {
+      "epoch": 0.628,
+      "grad_norm": 24.530260893855512,
+      "learning_rate": 3.222307917871977e-07,
+      "logits/chosen": -1.1015625,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -5.25,
+      "logps/rejected": -36.0,
+      "loss": 1.1445,
+      "nll_loss": 0.46875,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.53125,
+      "rewards/margins": 2.90625,
+      "rewards/rejected": -0.376953125,
+      "step": 157
+    },
+    {
+      "epoch": 0.632,
+      "grad_norm": 42.24994864204084,
+      "learning_rate": 3.161791516836399e-07,
+      "logits/chosen": -1.078125,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -6.28125,
+      "logps/rejected": -32.25,
+      "loss": 1.0581,
+      "nll_loss": 0.55859375,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.8125,
+      "rewards/margins": 2.328125,
+      "rewards/rejected": 0.484375,
+      "step": 158
+    },
+    {
+      "epoch": 0.636,
+      "grad_norm": 14.715119673174955,
+      "learning_rate": 3.1015848988714307e-07,
+      "logits/chosen": -1.0546875,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -6.5,
+      "logps/rejected": -33.75,
+      "loss": 0.9939,
+      "nll_loss": 0.609375,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 3.203125,
+      "rewards/margins": 2.90625,
+      "rewards/rejected": 0.296875,
+      "step": 159
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 20.14211721361461,
+      "learning_rate": 3.0416982102641487e-07,
+      "logits/chosen": -1.0625,
+      "logits/rejected": -1.0625,
+      "logps/chosen": -5.625,
+      "logps/rejected": -27.25,
+      "loss": 1.0876,
+      "nll_loss": 0.5703125,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 3.296875,
+      "rewards/margins": 2.390625,
+      "rewards/rejected": 0.8984375,
+      "step": 160
+    },
+    {
+      "epoch": 0.644,
+      "grad_norm": 15.676197977531485,
+      "learning_rate": 2.982141543385717e-07,
+      "logits/chosen": -1.0703125,
+      "logits/rejected": -1.0625,
+      "logps/chosen": -4.75,
+      "logps/rejected": -25.25,
+      "loss": 1.0416,
+      "nll_loss": 0.490234375,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 2.953125,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": 1.1015625,
+      "step": 161
+    },
+    {
+      "epoch": 0.648,
+      "grad_norm": 18.922807692403683,
+      "learning_rate": 2.922924934990568e-07,
+      "logits/chosen": -1.03125,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -4.21875,
+      "logps/rejected": -28.25,
+      "loss": 1.0214,
+      "nll_loss": 0.427734375,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 2.578125,
+      "rewards/margins": 2.1875,
+      "rewards/rejected": 0.3984375,
+      "step": 162
+    },
+    {
+      "epoch": 0.652,
+      "grad_norm": 15.211250498950244,
+      "learning_rate": 2.864058364524964e-07,
+      "logits/chosen": -1.0625,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -3.875,
+      "logps/rejected": -26.625,
+      "loss": 1.0185,
+      "nll_loss": 0.39453125,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 2.984375,
+      "rewards/margins": 2.453125,
+      "rewards/rejected": 0.53515625,
+      "step": 163
+    },
+    {
+      "epoch": 0.656,
+      "grad_norm": 32.56527905487464,
+      "learning_rate": 2.8055517524452215e-07,
+      "logits/chosen": -1.1171875,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -4.875,
+      "logps/rejected": -27.125,
+      "loss": 0.9807,
+      "nll_loss": 0.4453125,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.6875,
+      "rewards/margins": 2.09375,
+      "rewards/rejected": 0.58984375,
+      "step": 164
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 10.018587276474245,
+      "learning_rate": 2.7474149585458664e-07,
+      "logits/chosen": -1.09375,
+      "logits/rejected": -1.1171875,
+      "logps/chosen": -3.390625,
+      "logps/rejected": -32.75,
+      "loss": 1.0211,
+      "nll_loss": 0.310546875,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 2.484375,
+      "rewards/margins": 2.421875,
+      "rewards/rejected": 0.0595703125,
+      "step": 165
+    },
+    {
+      "epoch": 0.664,
+      "grad_norm": 37.10528373181994,
+      "learning_rate": 2.6896577802980184e-07,
+      "logits/chosen": -1.078125,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -3.296875,
+      "logps/rejected": -33.25,
+      "loss": 1.0286,
+      "nll_loss": 0.310546875,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 2.734375,
+      "rewards/margins": 2.71875,
+      "rewards/rejected": 0.0106201171875,
+      "step": 166
+    },
+    {
+      "epoch": 0.668,
+      "grad_norm": 17.20600574229811,
+      "learning_rate": 2.632289951198285e-07,
+      "logits/chosen": -1.0859375,
+      "logits/rejected": -1.109375,
+      "logps/chosen": -5.1875,
+      "logps/rejected": -30.375,
+      "loss": 1.1248,
+      "nll_loss": 0.47265625,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 3.078125,
+      "rewards/margins": 2.46875,
+      "rewards/rejected": 0.609375,
+      "step": 167
+    },
+    {
+      "epoch": 0.672,
+      "grad_norm": 38.3608360818198,
+      "learning_rate": 2.575321139128417e-07,
+      "logits/chosen": -1.1171875,
+      "logits/rejected": -1.1015625,
+      "logps/chosen": -5.25,
+      "logps/rejected": -26.75,
+      "loss": 0.9945,
+      "nll_loss": 0.49609375,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.984375,
+      "rewards/margins": 1.9765625,
+      "rewards/rejected": 1.0,
+      "step": 168
+    },
+    {
+      "epoch": 0.676,
+      "grad_norm": 17.759279183503754,
+      "learning_rate": 2.5187609447260417e-07,
+      "logits/chosen": -1.1015625,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -4.375,
+      "logps/rejected": -30.25,
+      "loss": 1.0025,
+      "nll_loss": 0.439453125,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 2.96875,
+      "rewards/margins": 2.625,
+      "rewards/rejected": 0.3515625,
+      "step": 169
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 14.507361507718214,
+      "learning_rate": 2.462618899766722e-07,
+      "logits/chosen": -1.0546875,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -6.21875,
+      "logps/rejected": -32.75,
+      "loss": 1.0907,
+      "nll_loss": 0.60546875,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.359375,
+      "rewards/margins": 2.078125,
+      "rewards/rejected": 0.2734375,
+      "step": 170
+    },
+    {
+      "epoch": 0.684,
+      "grad_norm": 22.962611189789666,
+      "learning_rate": 2.406904465557614e-07,
+      "logits/chosen": -1.109375,
+      "logits/rejected": -1.1171875,
+      "logps/chosen": -4.5625,
+      "logps/rejected": -28.25,
+      "loss": 1.0443,
+      "nll_loss": 0.4296875,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 2.671875,
+      "rewards/margins": 1.859375,
+      "rewards/rejected": 0.8203125,
+      "step": 171
+    },
+    {
+      "epoch": 0.688,
+      "grad_norm": 18.306407029166536,
+      "learning_rate": 2.3516270313430081e-07,
+      "logits/chosen": -1.1015625,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -7.21875,
+      "logps/rejected": -33.0,
+      "loss": 1.0611,
+      "nll_loss": 0.6484375,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.9375,
+      "rewards/margins": 2.796875,
+      "rewards/rejected": 0.140625,
+      "step": 172
+    },
+    {
+      "epoch": 0.692,
+      "grad_norm": 12.791171107272646,
+      "learning_rate": 2.2967959127220137e-07,
+      "logits/chosen": -1.0859375,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -3.765625,
+      "logps/rejected": -26.25,
+      "loss": 0.9565,
+      "nll_loss": 0.3515625,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 3.125,
+      "rewards/margins": 2.5,
+      "rewards/rejected": 0.6328125,
+      "step": 173
+    },
+    {
+      "epoch": 0.696,
+      "grad_norm": 24.89717076883767,
+      "learning_rate": 2.242420350078647e-07,
+      "logits/chosen": -1.0703125,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -5.1875,
+      "logps/rejected": -27.75,
+      "loss": 1.0301,
+      "nll_loss": 0.486328125,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 3.015625,
+      "rewards/margins": 2.421875,
+      "rewards/rejected": 0.59765625,
+      "step": 174
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 30.549643731351598,
+      "learning_rate": 2.1885095070246113e-07,
+      "logits/chosen": -1.1015625,
+      "logits/rejected": -1.109375,
+      "logps/chosen": -4.53125,
+      "logps/rejected": -34.0,
+      "loss": 1.1014,
+      "nll_loss": 0.439453125,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 3.046875,
+      "rewards/margins": 2.8125,
+      "rewards/rejected": 0.2412109375,
+      "step": 175
+    },
+    {
+      "epoch": 0.704,
+      "grad_norm": 42.66332171882467,
+      "learning_rate": 2.1350724688549904e-07,
+      "logits/chosen": -1.15625,
+      "logits/rejected": -1.1484375,
+      "logps/chosen": -5.46875,
+      "logps/rejected": -31.5,
+      "loss": 1.1445,
+      "nll_loss": 0.5625,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 2.390625,
+      "rewards/margins": 2.09375,
+      "rewards/rejected": 0.306640625,
+      "step": 176
+    },
+    {
+      "epoch": 0.708,
+      "grad_norm": 19.430831101872652,
+      "learning_rate": 2.0821182410171633e-07,
+      "logits/chosen": -1.109375,
+      "logits/rejected": -1.125,
+      "logps/chosen": -3.5625,
+      "logps/rejected": -32.75,
+      "loss": 1.0655,
+      "nll_loss": 0.328125,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.390625,
+      "rewards/margins": 2.171875,
+      "rewards/rejected": 0.2099609375,
+      "step": 177
+    },
+    {
+      "epoch": 0.712,
+      "grad_norm": 40.306067494211455,
+      "learning_rate": 2.0296557475931687e-07,
+      "logits/chosen": -1.09375,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -6.21875,
+      "logps/rejected": -28.0,
+      "loss": 1.1514,
+      "nll_loss": 0.59375,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.84375,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": 1.1796875,
+      "step": 178
+    },
+    {
+      "epoch": 0.716,
+      "grad_norm": 21.550210732610292,
+      "learning_rate": 1.9776938297957686e-07,
+      "logits/chosen": -1.0625,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -3.140625,
+      "logps/rejected": -18.75,
+      "loss": 1.0927,
+      "nll_loss": 0.33203125,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 3.0,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": 1.546875,
+      "step": 179
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 16.05057780811457,
+      "learning_rate": 1.9262412444784958e-07,
+      "logits/chosen": -1.0078125,
+      "logits/rejected": -1.0234375,
+      "logps/chosen": -7.0,
+      "logps/rejected": -21.25,
+      "loss": 1.0314,
+      "nll_loss": 0.6875,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.75,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": 1.2578125,
+      "step": 180
+    },
+    {
+      "epoch": 0.724,
+      "grad_norm": 22.311212494168778,
+      "learning_rate": 1.8753066626599084e-07,
+      "logits/chosen": -1.1015625,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -3.75,
+      "logps/rejected": -34.25,
+      "loss": 1.1563,
+      "nll_loss": 0.38671875,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.84375,
+      "rewards/margins": 2.234375,
+      "rewards/rejected": 0.6015625,
+      "step": 181
+    },
+    {
+      "epoch": 0.728,
+      "grad_norm": 24.732581587145912,
+      "learning_rate": 1.8248986680623074e-07,
+      "logits/chosen": -1.0625,
+      "logits/rejected": -1.0625,
+      "logps/chosen": -5.0625,
+      "logps/rejected": -23.625,
+      "loss": 1.1566,
+      "nll_loss": 0.5078125,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.59375,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": 1.046875,
+      "step": 182
+    },
+    {
+      "epoch": 0.732,
+      "grad_norm": 52.267590702636376,
+      "learning_rate": 1.7750257556651805e-07,
+      "logits/chosen": -1.078125,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -4.09375,
+      "logps/rejected": -31.0,
+      "loss": 1.1124,
+      "nll_loss": 0.384765625,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.796875,
+      "rewards/margins": 2.421875,
+      "rewards/rejected": 0.375,
+      "step": 183
+    },
+    {
+      "epoch": 0.736,
+      "grad_norm": 29.867593385467835,
+      "learning_rate": 1.725696330273575e-07,
+      "logits/chosen": -1.0390625,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -3.9375,
+      "logps/rejected": -27.625,
+      "loss": 1.0221,
+      "nll_loss": 0.41015625,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 3.09375,
+      "rewards/margins": 2.203125,
+      "rewards/rejected": 0.890625,
+      "step": 184
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 25.170290330540713,
+      "learning_rate": 1.6769187051016932e-07,
+      "logits/chosen": -1.0703125,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -3.90625,
+      "logps/rejected": -24.75,
+      "loss": 1.0845,
+      "nll_loss": 0.400390625,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.96875,
+      "rewards/margins": 2.046875,
+      "rewards/rejected": 0.90625,
+      "step": 185
+    },
+    {
+      "epoch": 0.744,
+      "grad_norm": 17.759260894082026,
+      "learning_rate": 1.6287011003719102e-07,
+      "logits/chosen": -1.1015625,
+      "logits/rejected": -1.109375,
+      "logps/chosen": -4.5,
+      "logps/rejected": -26.75,
+      "loss": 1.0185,
+      "nll_loss": 0.455078125,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 3.234375,
+      "rewards/margins": 2.640625,
+      "rewards/rejected": 0.6015625,
+      "step": 186
+    },
+    {
+      "epoch": 0.748,
+      "grad_norm": 33.46964477076418,
+      "learning_rate": 1.581051641929465e-07,
+      "logits/chosen": -1.1015625,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -5.40625,
+      "logps/rejected": -22.875,
+      "loss": 1.1376,
+      "nll_loss": 0.51953125,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.5625,
+      "rewards/margins": 1.5,
+      "rewards/rejected": 1.0625,
+      "step": 187
+    },
+    {
+      "epoch": 0.752,
+      "grad_norm": 24.562745533923515,
+      "learning_rate": 1.5339783598730565e-07,
+      "logits/chosen": -1.046875,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -5.5,
+      "logps/rejected": -27.0,
+      "loss": 1.1846,
+      "nll_loss": 0.52734375,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.640625,
+      "rewards/margins": 1.90625,
+      "rewards/rejected": 0.73828125,
+      "step": 188
+    },
+    {
+      "epoch": 0.756,
+      "grad_norm": 41.7355263163881,
+      "learning_rate": 1.4874891872015734e-07,
+      "logits/chosen": -1.046875,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -5.90625,
+      "logps/rejected": -25.25,
+      "loss": 1.0575,
+      "nll_loss": 0.546875,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.96875,
+      "rewards/margins": 2.078125,
+      "rewards/rejected": 0.89453125,
+      "step": 189
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 28.3791093205466,
+      "learning_rate": 1.4415919584771995e-07,
+      "logits/chosen": -1.0625,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -4.90625,
+      "logps/rejected": -25.75,
+      "loss": 1.1332,
+      "nll_loss": 0.48828125,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 2.40625,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": 0.92578125,
+      "step": 190
+    },
+    {
+      "epoch": 0.764,
+      "grad_norm": 19.154245446560328,
+      "learning_rate": 1.3962944085050832e-07,
+      "logits/chosen": -1.0234375,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -6.125,
+      "logps/rejected": -25.125,
+      "loss": 1.0726,
+      "nll_loss": 0.55078125,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.859375,
+      "rewards/margins": 1.921875,
+      "rewards/rejected": 0.93359375,
+      "step": 191
+    },
+    {
+      "epoch": 0.768,
+      "grad_norm": 17.938238931793443,
+      "learning_rate": 1.3516041710298497e-07,
+      "logits/chosen": -1.0859375,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -4.125,
+      "logps/rejected": -19.875,
+      "loss": 0.9941,
+      "nll_loss": 0.423828125,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.890625,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": 1.2890625,
+      "step": 192
+    },
+    {
+      "epoch": 0.772,
+      "grad_norm": 20.81847675232155,
+      "learning_rate": 1.3075287774491146e-07,
+      "logits/chosen": -1.0703125,
+      "logits/rejected": -1.1015625,
+      "logps/chosen": -2.65625,
+      "logps/rejected": -28.625,
+      "loss": 1.0433,
+      "nll_loss": 0.271484375,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 3.046875,
+      "rewards/margins": 2.203125,
+      "rewards/rejected": 0.83203125,
+      "step": 193
+    },
+    {
+      "epoch": 0.776,
+      "grad_norm": 44.286085577032274,
+      "learning_rate": 1.2640756555442683e-07,
+      "logits/chosen": -1.0703125,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -6.5625,
+      "logps/rejected": -25.125,
+      "loss": 1.0916,
+      "nll_loss": 0.64453125,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 3.078125,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": 1.1640625,
+      "step": 194
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 22.029622182069097,
+      "learning_rate": 1.2212521282287093e-07,
+      "logits/chosen": -1.0625,
+      "logits/rejected": -1.0625,
+      "logps/chosen": -4.5625,
+      "logps/rejected": -28.125,
+      "loss": 1.0946,
+      "nll_loss": 0.462890625,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 2.9375,
+      "rewards/margins": 2.0625,
+      "rewards/rejected": 0.87109375,
+      "step": 195
+    },
+    {
+      "epoch": 0.784,
+      "grad_norm": 11.861056714322848,
+      "learning_rate": 1.179065412313755e-07,
+      "logits/chosen": -1.1171875,
+      "logits/rejected": -1.1015625,
+      "logps/chosen": -6.5,
+      "logps/rejected": -21.75,
+      "loss": 1.0592,
+      "nll_loss": 0.640625,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 2.90625,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": 1.2265625,
+      "step": 196
+    },
+    {
+      "epoch": 0.788,
+      "grad_norm": 14.546735669018146,
+      "learning_rate": 1.137522617292434e-07,
+      "logits/chosen": -1.03125,
+      "logits/rejected": -1.0234375,
+      "logps/chosen": -4.34375,
+      "logps/rejected": -18.5,
+      "loss": 1.0507,
+      "nll_loss": 0.41796875,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 3.109375,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": 1.7890625,
+      "step": 197
+    },
+    {
+      "epoch": 0.792,
+      "grad_norm": 55.63915697780125,
+      "learning_rate": 1.0966307441413597e-07,
+      "logits/chosen": -1.1015625,
+      "logits/rejected": -1.1015625,
+      "logps/chosen": -6.125,
+      "logps/rejected": -23.25,
+      "loss": 1.0632,
+      "nll_loss": 0.60546875,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 3.140625,
+      "rewards/margins": 1.9921875,
+      "rewards/rejected": 1.140625,
+      "step": 198
+    },
+    {
+      "epoch": 0.796,
+      "grad_norm": 20.1761931612358,
+      "learning_rate": 1.0563966841408995e-07,
+      "logits/chosen": -1.0703125,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -5.03125,
+      "logps/rejected": -29.0,
+      "loss": 0.9663,
+      "nll_loss": 0.494140625,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 3.25,
+      "rewards/margins": 2.546875,
+      "rewards/rejected": 0.703125,
+      "step": 199
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 23.608862708071296,
+      "learning_rate": 1.0168272177138199e-07,
+      "logits/chosen": -1.0625,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -4.9375,
+      "logps/rejected": -28.5,
+      "loss": 1.0909,
+      "nll_loss": 0.466796875,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 2.90625,
+      "rewards/margins": 2.109375,
+      "rewards/rejected": 0.8046875,
+      "step": 200
+    },
+    {
+      "epoch": 0.804,
+      "grad_norm": 41.55024587771104,
+      "learning_rate": 9.779290132826224e-08,
+      "logits/chosen": -1.09375,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -4.84375,
+      "logps/rejected": -24.125,
+      "loss": 1.0829,
+      "nll_loss": 0.44140625,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 3.390625,
+      "rewards/margins": 1.9453125,
+      "rewards/rejected": 1.4453125,
+      "step": 201
+    },
+    {
+      "epoch": 0.808,
+      "grad_norm": 34.54633889712777,
+      "learning_rate": 9.39708626145751e-08,
+      "logits/chosen": -1.0546875,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -5.90625,
+      "logps/rejected": -27.25,
+      "loss": 1.0733,
+      "nll_loss": 0.57421875,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 3.28125,
+      "rewards/margins": 2.203125,
+      "rewards/rejected": 1.078125,
+      "step": 202
+    },
+    {
+      "epoch": 0.812,
+      "grad_norm": 32.77492433560517,
+      "learning_rate": 9.021724973728634e-08,
+      "logits/chosen": -1.0546875,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -4.875,
+      "logps/rejected": -31.75,
+      "loss": 1.0815,
+      "nll_loss": 0.478515625,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.921875,
+      "rewards/margins": 2.59375,
+      "rewards/rejected": 0.318359375,
+      "step": 203
+    },
+    {
+      "epoch": 0.816,
+      "grad_norm": 12.483799623280103,
+      "learning_rate": 8.653269527193568e-08,
+      "logits/chosen": -1.109375,
+      "logits/rejected": -1.109375,
+      "logps/chosen": -5.25,
+      "logps/rejected": -25.375,
+      "loss": 1.0666,
+      "nll_loss": 0.486328125,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 3.0,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": 1.0859375,
+      "step": 204
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 52.55865885582505,
+      "learning_rate": 8.291782015603176e-08,
+      "logits/chosen": -1.140625,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -6.0625,
+      "logps/rejected": -20.375,
+      "loss": 1.1337,
+      "nll_loss": 0.63671875,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 2.625,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": 1.359375,
+      "step": 205
+    },
+    {
+      "epoch": 0.824,
+      "grad_norm": 32.755044229250124,
+      "learning_rate": 7.937323358440934e-08,
+      "logits/chosen": -1.0703125,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -4.0625,
+      "logps/rejected": -22.875,
+      "loss": 1.0425,
+      "nll_loss": 0.404296875,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 3.203125,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": 1.484375,
+      "step": 206
+    },
+    {
+      "epoch": 0.828,
+      "grad_norm": 49.57046953822534,
+      "learning_rate": 7.589953290656532e-08,
+      "logits/chosen": -1.1171875,
+      "logits/rejected": -1.1015625,
+      "logps/chosen": -3.234375,
+      "logps/rejected": -28.25,
+      "loss": 1.1414,
+      "nll_loss": 0.32421875,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 2.859375,
+      "rewards/margins": 2.4375,
+      "rewards/rejected": 0.435546875,
+      "step": 207
+    },
+    {
+      "epoch": 0.832,
+      "grad_norm": 29.02033137242745,
+      "learning_rate": 7.249730352598998e-08,
+      "logits/chosen": -1.09375,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -4.8125,
+      "logps/rejected": -28.75,
+      "loss": 1.004,
+      "nll_loss": 0.451171875,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.671875,
+      "rewards/margins": 2.171875,
+      "rewards/rejected": 0.51171875,
+      "step": 208
+    },
+    {
+      "epoch": 0.836,
+      "grad_norm": 17.714740130325648,
+      "learning_rate": 6.916711880151304e-08,
+      "logits/chosen": -1.109375,
+      "logits/rejected": -1.1015625,
+      "logps/chosen": -6.25,
+      "logps/rejected": -30.125,
+      "loss": 1.1389,
+      "nll_loss": 0.609375,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 2.703125,
+      "rewards/margins": 2.109375,
+      "rewards/rejected": 0.58984375,
+      "step": 209
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 17.72865807824904,
+      "learning_rate": 6.59095399506781e-08,
+      "logits/chosen": -1.1015625,
+      "logits/rejected": -1.1015625,
+      "logps/chosen": -6.5625,
+      "logps/rejected": -26.5,
+      "loss": 1.1747,
+      "nll_loss": 0.6484375,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.8125,
+      "rewards/margins": 1.875,
+      "rewards/rejected": 0.94140625,
+      "step": 210
+    },
+    {
+      "epoch": 0.844,
+      "grad_norm": 13.99041401845889,
+      "learning_rate": 6.2725115955164e-08,
+      "logits/chosen": -1.09375,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -4.1875,
+      "logps/rejected": -33.5,
+      "loss": 1.0243,
+      "nll_loss": 0.3984375,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 2.96875,
+      "rewards/margins": 2.828125,
+      "rewards/rejected": 0.140625,
+      "step": 211
+    },
+    {
+      "epoch": 0.848,
+      "grad_norm": 15.906985339916742,
+      "learning_rate": 5.961438346826792e-08,
+      "logits/chosen": -1.1015625,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -4.625,
+      "logps/rejected": -28.0,
+      "loss": 1.0347,
+      "nll_loss": 0.443359375,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.59375,
+      "rewards/margins": 2.03125,
+      "rewards/rejected": 0.55859375,
+      "step": 212
+    },
+    {
+      "epoch": 0.852,
+      "grad_norm": 36.27762496758833,
+      "learning_rate": 5.6577866724466296e-08,
+      "logits/chosen": -1.140625,
+      "logits/rejected": -1.1171875,
+      "logps/chosen": -6.53125,
+      "logps/rejected": -47.75,
+      "loss": 1.0298,
+      "nll_loss": 0.58203125,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 2.328125,
+      "rewards/margins": 3.59375,
+      "rewards/rejected": -1.265625,
+      "step": 213
+    },
+    {
+      "epoch": 0.856,
+      "grad_norm": 22.188902025969416,
+      "learning_rate": 5.361607745106816e-08,
+      "logits/chosen": -1.078125,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -3.734375,
+      "logps/rejected": -22.5,
+      "loss": 1.0263,
+      "nll_loss": 0.431640625,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 3.0,
+      "rewards/margins": 1.9375,
+      "rewards/rejected": 1.0625,
+      "step": 214
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 26.514396481742928,
+      "learning_rate": 5.072951478197723e-08,
+      "logits/chosen": -1.1015625,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -4.46875,
+      "logps/rejected": -31.5,
+      "loss": 1.0853,
+      "nll_loss": 0.4375,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 2.8125,
+      "rewards/margins": 2.296875,
+      "rewards/rejected": 0.515625,
+      "step": 215
+    },
+    {
+      "epoch": 0.864,
+      "grad_norm": 39.18691508367433,
+      "learning_rate": 4.79186651735749e-08,
+      "logits/chosen": -1.0703125,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -4.25,
+      "logps/rejected": -23.625,
+      "loss": 0.9658,
+      "nll_loss": 0.427734375,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 2.828125,
+      "rewards/margins": 1.625,
+      "rewards/rejected": 1.2109375,
+      "step": 216
+    },
+    {
+      "epoch": 0.868,
+      "grad_norm": 17.053221125940922,
+      "learning_rate": 4.518400232274078e-08,
+      "logits/chosen": -1.1171875,
+      "logits/rejected": -1.109375,
+      "logps/chosen": -4.90625,
+      "logps/rejected": -27.125,
+      "loss": 1.0356,
+      "nll_loss": 0.4375,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.65625,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": 0.7421875,
+      "step": 217
+    },
+    {
+      "epoch": 0.872,
+      "grad_norm": 30.04204392176379,
+      "learning_rate": 4.252598708702343e-08,
+      "logits/chosen": -1.0625,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -5.375,
+      "logps/rejected": -24.25,
+      "loss": 1.083,
+      "nll_loss": 0.53125,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.765625,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": 1.203125,
+      "step": 218
+    },
+    {
+      "epoch": 0.876,
+      "grad_norm": 12.813171720804812,
+      "learning_rate": 3.994506740697406e-08,
+      "logits/chosen": -1.046875,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -4.5625,
+      "logps/rejected": -23.0,
+      "loss": 0.9825,
+      "nll_loss": 0.419921875,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.734375,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": 1.15625,
+      "step": 219
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 31.189313546893203,
+      "learning_rate": 3.744167823065814e-08,
+      "logits/chosen": -1.0859375,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -4.96875,
+      "logps/rejected": -36.0,
+      "loss": 1.0805,
+      "nll_loss": 0.478515625,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.296875,
+      "rewards/margins": 2.71875,
+      "rewards/rejected": -0.4296875,
+      "step": 220
+    },
+    {
+      "epoch": 0.884,
+      "grad_norm": 38.49107579612584,
+      "learning_rate": 3.501624144035559e-08,
+      "logits/chosen": -1.1015625,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -5.5,
+      "logps/rejected": -23.375,
+      "loss": 1.097,
+      "nll_loss": 0.53125,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.78125,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": 1.09375,
+      "step": 221
+    },
+    {
+      "epoch": 0.888,
+      "grad_norm": 13.5024131062282,
+      "learning_rate": 3.2669165781463395e-08,
+      "logits/chosen": -1.0859375,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -4.53125,
+      "logps/rejected": -29.875,
+      "loss": 1.0892,
+      "nll_loss": 0.44921875,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.765625,
+      "rewards/margins": 1.625,
+      "rewards/rejected": 1.140625,
+      "step": 222
+    },
+    {
+      "epoch": 0.892,
+      "grad_norm": 28.282704129587508,
+      "learning_rate": 3.0400846793612544e-08,
+      "logits/chosen": -1.0859375,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -4.71875,
+      "logps/rejected": -23.25,
+      "loss": 1.0451,
+      "nll_loss": 0.462890625,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.859375,
+      "rewards/margins": 1.28125,
+      "rewards/rejected": 1.578125,
+      "step": 223
+    },
+    {
+      "epoch": 0.896,
+      "grad_norm": 37.60742006233887,
+      "learning_rate": 2.8211666744009045e-08,
+      "logits/chosen": -1.1328125,
+      "logits/rejected": -1.125,
+      "logps/chosen": -3.796875,
+      "logps/rejected": -29.875,
+      "loss": 1.0323,
+      "nll_loss": 0.333984375,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.671875,
+      "rewards/margins": 2.265625,
+      "rewards/rejected": 0.41015625,
+      "step": 224
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 29.516319250938352,
+      "learning_rate": 2.6101994563013098e-08,
+      "logits/chosen": -1.078125,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -3.453125,
+      "logps/rejected": -26.25,
+      "loss": 1.0479,
+      "nll_loss": 0.341796875,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 2.59375,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": 0.7890625,
+      "step": 225
+    },
+    {
+      "epoch": 0.904,
+      "grad_norm": 14.847458095289273,
+      "learning_rate": 2.4072185781965238e-08,
+      "logits/chosen": -1.0859375,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -4.8125,
+      "logps/rejected": -29.25,
+      "loss": 1.0546,
+      "nll_loss": 0.43359375,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 2.9375,
+      "rewards/margins": 2.203125,
+      "rewards/rejected": 0.7265625,
+      "step": 226
+    },
+    {
+      "epoch": 0.908,
+      "grad_norm": 17.36079249416162,
+      "learning_rate": 2.212258247327059e-08,
+      "logits/chosen": -1.109375,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -3.578125,
+      "logps/rejected": -31.875,
+      "loss": 1.0537,
+      "nll_loss": 0.359375,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.765625,
+      "rewards/margins": 2.34375,
+      "rewards/rejected": 0.4375,
+      "step": 227
+    },
+    {
+      "epoch": 0.912,
+      "grad_norm": 18.768525578098558,
+      "learning_rate": 2.025351319275137e-08,
+      "logits/chosen": -1.1015625,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -6.59375,
+      "logps/rejected": -38.25,
+      "loss": 1.106,
+      "nll_loss": 0.56640625,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.875,
+      "rewards/margins": 3.203125,
+      "rewards/rejected": -0.31640625,
+      "step": 228
+    },
+    {
+      "epoch": 0.916,
+      "grad_norm": 33.5917307298862,
+      "learning_rate": 1.8465292924276844e-08,
+      "logits/chosen": -1.1484375,
+      "logits/rejected": -1.1484375,
+      "logps/chosen": -5.09375,
+      "logps/rejected": -42.75,
+      "loss": 0.9739,
+      "nll_loss": 0.46875,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 3.078125,
+      "rewards/margins": 3.578125,
+      "rewards/rejected": -0.50390625,
+      "step": 229
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 12.955837772485086,
+      "learning_rate": 1.6758223026681507e-08,
+      "logits/chosen": -1.0703125,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -6.1875,
+      "logps/rejected": -30.75,
+      "loss": 1.0963,
+      "nll_loss": 0.58203125,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.734375,
+      "rewards/margins": 2.234375,
+      "rewards/rejected": 0.5,
+      "step": 230
+    },
+    {
+      "epoch": 0.924,
+      "grad_norm": 16.2992963110193,
+      "learning_rate": 1.5132591182978105e-08,
+      "logits/chosen": -1.0625,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -6.28125,
+      "logps/rejected": -31.875,
+      "loss": 1.0419,
+      "nll_loss": 0.56640625,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 2.75,
+      "rewards/margins": 2.265625,
+      "rewards/rejected": 0.482421875,
+      "step": 231
+    },
+    {
+      "epoch": 0.928,
+      "grad_norm": 12.293312024242535,
+      "learning_rate": 1.3588671351876357e-08,
+      "logits/chosen": -1.0703125,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -4.53125,
+      "logps/rejected": -22.875,
+      "loss": 1.0256,
+      "nll_loss": 0.439453125,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 2.59375,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": 1.15625,
+      "step": 232
+    },
+    {
+      "epoch": 0.932,
+      "grad_norm": 18.415969639868365,
+      "learning_rate": 1.2126723721614052e-08,
+      "logits/chosen": -1.0859375,
+      "logits/rejected": -1.109375,
+      "logps/chosen": -4.71875,
+      "logps/rejected": -28.5,
+      "loss": 1.0731,
+      "nll_loss": 0.46484375,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.59375,
+      "rewards/margins": 2.203125,
+      "rewards/rejected": 0.376953125,
+      "step": 233
+    },
+    {
+      "epoch": 0.936,
+      "grad_norm": 17.63813786850423,
+      "learning_rate": 1.0746994666109233e-08,
+      "logits/chosen": -1.109375,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -4.75,
+      "logps/rejected": -20.5,
+      "loss": 0.9922,
+      "nll_loss": 0.431640625,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 3.46875,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": 1.6953125,
+      "step": 234
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 15.498735501179883,
+      "learning_rate": 9.449716703439803e-09,
+      "logits/chosen": -1.1171875,
+      "logits/rejected": -1.140625,
+      "logps/chosen": -4.6875,
+      "logps/rejected": -23.625,
+      "loss": 1.0851,
+      "nll_loss": 0.404296875,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.9375,
+      "rewards/margins": 2.046875,
+      "rewards/rejected": 0.89453125,
+      "step": 235
+    },
+    {
+      "epoch": 0.944,
+      "grad_norm": 11.978784225505468,
+      "learning_rate": 8.235108456658812e-09,
+      "logits/chosen": -1.140625,
+      "logits/rejected": -1.1328125,
+      "logps/chosen": -4.03125,
+      "logps/rejected": -39.25,
+      "loss": 1.0219,
+      "nll_loss": 0.390625,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 2.59375,
+      "rewards/margins": 3.0,
+      "rewards/rejected": -0.41015625,
+      "step": 236
+    },
+    {
+      "epoch": 0.948,
+      "grad_norm": 13.770414732547914,
+      "learning_rate": 7.103374616951041e-09,
+      "logits/chosen": -1.0546875,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -3.25,
+      "logps/rejected": -21.75,
+      "loss": 1.0617,
+      "nll_loss": 0.322265625,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 3.3125,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": 1.6015625,
+      "step": 237
+    },
+    {
+      "epoch": 0.952,
+      "grad_norm": 26.345272412615557,
+      "learning_rate": 6.054705909137425e-09,
+      "logits/chosen": -1.1015625,
+      "logits/rejected": -1.125,
+      "logps/chosen": -4.1875,
+      "logps/rejected": -38.0,
+      "loss": 1.078,
+      "nll_loss": 0.443359375,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 2.734375,
+      "rewards/margins": 3.03125,
+      "rewards/rejected": -0.30078125,
+      "step": 238
+    },
+    {
+      "epoch": 0.956,
+      "grad_norm": 17.86401941431699,
+      "learning_rate": 5.0892790595336575e-09,
+      "logits/chosen": -1.125,
+      "logits/rejected": -1.1328125,
+      "logps/chosen": -4.4375,
+      "logps/rejected": -27.5,
+      "loss": 0.9794,
+      "nll_loss": 0.451171875,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 2.640625,
+      "rewards/margins": 2.140625,
+      "rewards/rejected": 0.49609375,
+      "step": 239
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 17.07158211116171,
+      "learning_rate": 4.207256766166845e-09,
+      "logits/chosen": -1.078125,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -4.90625,
+      "logps/rejected": -26.625,
+      "loss": 1.1286,
+      "nll_loss": 0.443359375,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.453125,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": 0.875,
+      "step": 240
+    },
+    {
+      "epoch": 0.964,
+      "grad_norm": 40.38091842339886,
+      "learning_rate": 3.4087876713574935e-09,
+      "logits/chosen": -1.1015625,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -5.96875,
+      "logps/rejected": -31.375,
+      "loss": 0.9722,
+      "nll_loss": 0.609375,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 3.015625,
+      "rewards/margins": 2.421875,
+      "rewards/rejected": 0.58984375,
+      "step": 241
+    },
+    {
+      "epoch": 0.968,
+      "grad_norm": 14.247494421593556,
+      "learning_rate": 2.69400633666933e-09,
+      "logits/chosen": -1.1171875,
+      "logits/rejected": -1.109375,
+      "logps/chosen": -4.4375,
+      "logps/rejected": -21.375,
+      "loss": 1.0203,
+      "nll_loss": 0.408203125,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.71875,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": 1.078125,
+      "step": 242
+    },
+    {
+      "epoch": 0.972,
+      "grad_norm": 30.909416470072028,
+      "learning_rate": 2.0630332202324906e-09,
+      "logits/chosen": -1.0546875,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -5.65625,
+      "logps/rejected": -34.25,
+      "loss": 1.0594,
+      "nll_loss": 0.5,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 3.03125,
+      "rewards/margins": 2.96875,
+      "rewards/rejected": 0.06396484375,
+      "step": 243
+    },
+    {
+      "epoch": 0.976,
+      "grad_norm": 13.79697350875317,
+      "learning_rate": 1.5159746564433196e-09,
+      "logits/chosen": -1.09375,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -5.875,
+      "logps/rejected": -31.625,
+      "loss": 1.0882,
+      "nll_loss": 0.59375,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.625,
+      "rewards/margins": 2.5,
+      "rewards/rejected": 0.1337890625,
+      "step": 244
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 26.292855916544468,
+      "learning_rate": 1.0529228380445899e-09,
+      "logits/chosen": -1.125,
+      "logits/rejected": -1.1015625,
+      "logps/chosen": -4.9375,
+      "logps/rejected": -36.0,
+      "loss": 1.0991,
+      "nll_loss": 0.4453125,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 2.921875,
+      "rewards/margins": 2.765625,
+      "rewards/rejected": 0.1630859375,
+      "step": 245
+    },
+    {
+      "epoch": 0.984,
+      "grad_norm": 9.993888997764955,
+      "learning_rate": 6.739558005884883e-10,
+      "logits/chosen": -1.0625,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -6.125,
+      "logps/rejected": -25.625,
+      "loss": 1.0262,
+      "nll_loss": 0.52734375,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.78125,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": 1.109375,
+      "step": 246
+    },
+    {
+      "epoch": 0.988,
+      "grad_norm": 26.071889768691495,
+      "learning_rate": 3.791374092859678e-10,
+      "logits/chosen": -1.1171875,
+      "logits/rejected": -1.1328125,
+      "logps/chosen": -6.34375,
+      "logps/rejected": -31.75,
+      "loss": 1.0238,
+      "nll_loss": 0.55078125,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.515625,
+      "rewards/margins": 2.046875,
+      "rewards/rejected": 0.46875,
+      "step": 247
+    },
+    {
+      "epoch": 0.992,
+      "grad_norm": 27.845797479287505,
+      "learning_rate": 1.6851734824380182e-10,
+      "logits/chosen": -1.0703125,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -3.578125,
+      "logps/rejected": -32.75,
+      "loss": 1.1748,
+      "nll_loss": 0.341796875,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.953125,
+      "rewards/margins": 2.265625,
+      "rewards/rejected": 0.6875,
+      "step": 248
+    },
+    {
+      "epoch": 0.996,
+      "grad_norm": 16.83674597083853,
+      "learning_rate": 4.213111209155906e-11,
+      "logits/chosen": -1.140625,
+      "logits/rejected": -1.125,
+      "logps/chosen": -4.1875,
+      "logps/rejected": -30.125,
+      "loss": 0.9346,
+      "nll_loss": 0.38671875,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.984375,
+      "rewards/margins": 2.625,
+      "rewards/rejected": 0.35546875,
+      "step": 249
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 24.975249728436633,
+      "learning_rate": 0.0,
+      "logits/chosen": -1.09375,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -5.1875,
+      "logps/rejected": -30.875,
+      "loss": 1.0326,
+      "nll_loss": 0.5078125,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.375,
+      "rewards/margins": 2.171875,
+      "rewards/rejected": 0.2060546875,
+      "step": 250
+    },
+    {
+      "epoch": 1.0,
+      "step": 250,
+      "total_flos": 1.517913978562874e+20,
+      "train_loss": 1.4306324501037597,
+      "train_runtime": 16604.645,
+      "train_samples_per_second": 3.854,
+      "train_steps_per_second": 0.015
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 250,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.517913978562874e+20,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}