| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 100, | |
| "global_step": 37, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "debug/policy_chosen_logits": -3.41206693649292, | |
| "debug/policy_chosen_logps": -143.21307373046875, | |
| "debug/policy_rejected_logits": -3.2551214694976807, | |
| "debug/policy_rejected_logps": -150.2328338623047, | |
| "debug/reference_chosen_logps": -143.21307373046875, | |
| "debug/reference_rejected_logps": -150.2328338623047, | |
| "epoch": 0.02702702702702703, | |
| "grad_norm": 3.5140838495338116, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.41206693649292, | |
| "logits/rejected": -3.2551214694976807, | |
| "logps/chosen": -143.21307373046875, | |
| "logps/rejected": -150.2328338623047, | |
| "loss": 0.5, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.4049649238586426, | |
| "debug/policy_chosen_logps": -143.7670135498047, | |
| "debug/policy_rejected_logits": -3.4008069038391113, | |
| "debug/policy_rejected_logps": -138.0754852294922, | |
| "debug/reference_chosen_logps": -143.82278442382812, | |
| "debug/reference_rejected_logps": -138.1428985595703, | |
| "epoch": 0.05405405405405406, | |
| "grad_norm": 3.526231895119079, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.4049649238586426, | |
| "logits/rejected": -3.4008069038391113, | |
| "logps/chosen": -143.7670135498047, | |
| "logps/rejected": -138.0754852294922, | |
| "loss": 0.4994, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": 0.0005577945848926902, | |
| "rewards/margins": -0.00011638639261946082, | |
| "rewards/rejected": 0.000674180977512151, | |
| "step": 2 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.2883780002593994, | |
| "debug/policy_chosen_logps": -143.35385131835938, | |
| "debug/policy_rejected_logits": -3.2103564739227295, | |
| "debug/policy_rejected_logps": -139.85372924804688, | |
| "debug/reference_chosen_logps": -143.38311767578125, | |
| "debug/reference_rejected_logps": -139.98326110839844, | |
| "epoch": 0.08108108108108109, | |
| "grad_norm": 3.618148106626628, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.2883780002593994, | |
| "logits/rejected": -3.2103564739227295, | |
| "logps/chosen": -143.35385131835938, | |
| "logps/rejected": -139.85372924804688, | |
| "loss": 0.5006, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": 0.00029258732683956623, | |
| "rewards/margins": -0.001002683537080884, | |
| "rewards/rejected": 0.0012952708639204502, | |
| "step": 3 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.294776678085327, | |
| "debug/policy_chosen_logps": -132.79519653320312, | |
| "debug/policy_rejected_logits": -3.4150912761688232, | |
| "debug/policy_rejected_logps": -138.08180236816406, | |
| "debug/reference_chosen_logps": -132.62167358398438, | |
| "debug/reference_rejected_logps": -137.927978515625, | |
| "epoch": 0.10810810810810811, | |
| "grad_norm": 3.5093502617713077, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.294776678085327, | |
| "logits/rejected": -3.4150912761688232, | |
| "logps/chosen": -132.79519653320312, | |
| "logps/rejected": -138.08180236816406, | |
| "loss": 0.4994, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0017352199647575617, | |
| "rewards/margins": -0.0001968955621123314, | |
| "rewards/rejected": -0.0015383241698145866, | |
| "step": 4 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.343012809753418, | |
| "debug/policy_chosen_logps": -126.95188903808594, | |
| "debug/policy_rejected_logits": -3.3124771118164062, | |
| "debug/policy_rejected_logps": -129.9374237060547, | |
| "debug/reference_chosen_logps": -126.82910919189453, | |
| "debug/reference_rejected_logps": -129.91493225097656, | |
| "epoch": 0.13513513513513514, | |
| "grad_norm": 3.250245901043351, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.343012809753418, | |
| "logits/rejected": -3.3124771118164062, | |
| "logps/chosen": -126.95188903808594, | |
| "logps/rejected": -129.9374237060547, | |
| "loss": 0.5009, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.001227750675752759, | |
| "rewards/margins": -0.001002778997644782, | |
| "rewards/rejected": -0.00022497176541946828, | |
| "step": 5 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.2374277114868164, | |
| "debug/policy_chosen_logps": -154.26406860351562, | |
| "debug/policy_rejected_logits": -3.2978382110595703, | |
| "debug/policy_rejected_logps": -143.40101623535156, | |
| "debug/reference_chosen_logps": -153.9961395263672, | |
| "debug/reference_rejected_logps": -143.3165283203125, | |
| "epoch": 0.16216216216216217, | |
| "grad_norm": 3.439711062354016, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.2374277114868164, | |
| "logits/rejected": -3.2978382110595703, | |
| "logps/chosen": -154.26406860351562, | |
| "logps/rejected": -143.40101623535156, | |
| "loss": 0.5009, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.002679176162928343, | |
| "rewards/margins": -0.0018340873066335917, | |
| "rewards/rejected": -0.000845088972710073, | |
| "step": 6 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.2252252101898193, | |
| "debug/policy_chosen_logps": -162.3474578857422, | |
| "debug/policy_rejected_logits": -3.230537176132202, | |
| "debug/policy_rejected_logps": -147.3297576904297, | |
| "debug/reference_chosen_logps": -162.26889038085938, | |
| "debug/reference_rejected_logps": -147.57199096679688, | |
| "epoch": 0.1891891891891892, | |
| "grad_norm": 3.5128248367150468, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.2252252101898193, | |
| "logits/rejected": -3.230537176132202, | |
| "logps/chosen": -162.3474578857422, | |
| "logps/rejected": -147.3297576904297, | |
| "loss": 0.5005, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": -0.0007855510921217501, | |
| "rewards/margins": -0.003207969479262829, | |
| "rewards/rejected": 0.0024224184453487396, | |
| "step": 7 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.3006181716918945, | |
| "debug/policy_chosen_logps": -149.18630981445312, | |
| "debug/policy_rejected_logits": -3.3146488666534424, | |
| "debug/policy_rejected_logps": -147.72348022460938, | |
| "debug/reference_chosen_logps": -149.20960998535156, | |
| "debug/reference_rejected_logps": -147.54893493652344, | |
| "epoch": 0.21621621621621623, | |
| "grad_norm": 3.499360817762659, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.3006181716918945, | |
| "logits/rejected": -3.3146488666534424, | |
| "logps/chosen": -149.18630981445312, | |
| "logps/rejected": -147.72348022460938, | |
| "loss": 0.4992, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.000233030179515481, | |
| "rewards/margins": 0.001978444866836071, | |
| "rewards/rejected": -0.00174541468732059, | |
| "step": 8 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.3503048419952393, | |
| "debug/policy_chosen_logps": -150.99197387695312, | |
| "debug/policy_rejected_logits": -3.2822837829589844, | |
| "debug/policy_rejected_logps": -145.26345825195312, | |
| "debug/reference_chosen_logps": -151.06939697265625, | |
| "debug/reference_rejected_logps": -145.52816772460938, | |
| "epoch": 0.24324324324324326, | |
| "grad_norm": 3.4264003876223215, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.3503048419952393, | |
| "logits/rejected": -3.2822837829589844, | |
| "logps/chosen": -150.99197387695312, | |
| "logps/rejected": -145.26345825195312, | |
| "loss": 0.4987, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": 0.0007742022862657905, | |
| "rewards/margins": -0.0018729494186118245, | |
| "rewards/rejected": 0.002647151704877615, | |
| "step": 9 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.304420232772827, | |
| "debug/policy_chosen_logps": -133.21044921875, | |
| "debug/policy_rejected_logits": -3.412870168685913, | |
| "debug/policy_rejected_logps": -125.90888977050781, | |
| "debug/reference_chosen_logps": -133.26504516601562, | |
| "debug/reference_rejected_logps": -125.95523834228516, | |
| "epoch": 0.2702702702702703, | |
| "grad_norm": 3.3820605783151074, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.304420232772827, | |
| "logits/rejected": -3.412870168685913, | |
| "logps/chosen": -133.21044921875, | |
| "logps/rejected": -125.90888977050781, | |
| "loss": 0.5005, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.0005457591032609344, | |
| "rewards/margins": 8.22543297545053e-05, | |
| "rewards/rejected": 0.0004635047516785562, | |
| "step": 10 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.3585329055786133, | |
| "debug/policy_chosen_logps": -139.380859375, | |
| "debug/policy_rejected_logits": -3.3599538803100586, | |
| "debug/policy_rejected_logps": -132.3596649169922, | |
| "debug/reference_chosen_logps": -139.71542358398438, | |
| "debug/reference_rejected_logps": -132.61981201171875, | |
| "epoch": 0.2972972972972973, | |
| "grad_norm": 3.288218575224273, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.3585329055786133, | |
| "logits/rejected": -3.3599538803100586, | |
| "logps/chosen": -139.380859375, | |
| "logps/rejected": -132.3596649169922, | |
| "loss": 0.4999, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.0033456420060247183, | |
| "rewards/margins": 0.0007441901252605021, | |
| "rewards/rejected": 0.0026014517061412334, | |
| "step": 11 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.2882096767425537, | |
| "debug/policy_chosen_logps": -156.53334045410156, | |
| "debug/policy_rejected_logits": -3.2960500717163086, | |
| "debug/policy_rejected_logps": -143.13995361328125, | |
| "debug/reference_chosen_logps": -156.6325225830078, | |
| "debug/reference_rejected_logps": -142.83905029296875, | |
| "epoch": 0.32432432432432434, | |
| "grad_norm": 3.813316903837083, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.2882096767425537, | |
| "logits/rejected": -3.2960500717163086, | |
| "logps/chosen": -156.53334045410156, | |
| "logps/rejected": -143.13995361328125, | |
| "loss": 0.4989, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.0009917640127241611, | |
| "rewards/margins": 0.004000749439001083, | |
| "rewards/rejected": -0.0030089854262769222, | |
| "step": 12 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.2771499156951904, | |
| "debug/policy_chosen_logps": -133.85403442382812, | |
| "debug/policy_rejected_logits": -3.3577096462249756, | |
| "debug/policy_rejected_logps": -145.81051635742188, | |
| "debug/reference_chosen_logps": -134.21636962890625, | |
| "debug/reference_rejected_logps": -145.71588134765625, | |
| "epoch": 0.35135135135135137, | |
| "grad_norm": 3.5238547026932254, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.2771499156951904, | |
| "logits/rejected": -3.3577096462249756, | |
| "logps/chosen": -133.85403442382812, | |
| "logps/rejected": -145.81051635742188, | |
| "loss": 0.5004, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.003623266238719225, | |
| "rewards/margins": 0.004569797310978174, | |
| "rewards/rejected": -0.0009465311886742711, | |
| "step": 13 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.4804084300994873, | |
| "debug/policy_chosen_logps": -165.59515380859375, | |
| "debug/policy_rejected_logits": -3.5207834243774414, | |
| "debug/policy_rejected_logps": -139.4119873046875, | |
| "debug/reference_chosen_logps": -165.61618041992188, | |
| "debug/reference_rejected_logps": -139.546142578125, | |
| "epoch": 0.3783783783783784, | |
| "grad_norm": 3.2715325631312377, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.4804084300994873, | |
| "logits/rejected": -3.5207834243774414, | |
| "logps/chosen": -165.59515380859375, | |
| "logps/rejected": -139.4119873046875, | |
| "loss": 0.4997, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.00021032325457781553, | |
| "rewards/margins": -0.0011312393471598625, | |
| "rewards/rejected": 0.0013415624853223562, | |
| "step": 14 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.3723671436309814, | |
| "debug/policy_chosen_logps": -138.40748596191406, | |
| "debug/policy_rejected_logits": -3.2726492881774902, | |
| "debug/policy_rejected_logps": -146.87710571289062, | |
| "debug/reference_chosen_logps": -138.04966735839844, | |
| "debug/reference_rejected_logps": -146.4534454345703, | |
| "epoch": 0.40540540540540543, | |
| "grad_norm": 3.5479335407438923, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.3723671436309814, | |
| "logits/rejected": -3.2726492881774902, | |
| "logps/chosen": -138.40748596191406, | |
| "logps/rejected": -146.87710571289062, | |
| "loss": 0.4971, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.003578138304874301, | |
| "rewards/margins": 0.0006584453512914479, | |
| "rewards/rejected": -0.004236583597958088, | |
| "step": 15 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.280038356781006, | |
| "debug/policy_chosen_logps": -144.99691772460938, | |
| "debug/policy_rejected_logits": -3.265355110168457, | |
| "debug/policy_rejected_logps": -136.1260223388672, | |
| "debug/reference_chosen_logps": -145.38519287109375, | |
| "debug/reference_rejected_logps": -136.1790313720703, | |
| "epoch": 0.43243243243243246, | |
| "grad_norm": 3.3905692790559057, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.280038356781006, | |
| "logits/rejected": -3.265355110168457, | |
| "logps/chosen": -144.99691772460938, | |
| "logps/rejected": -136.1260223388672, | |
| "loss": 0.4987, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.0038827608805149794, | |
| "rewards/margins": 0.0033527181949466467, | |
| "rewards/rejected": 0.0005300427437759936, | |
| "step": 16 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.4201347827911377, | |
| "debug/policy_chosen_logps": -154.7392578125, | |
| "debug/policy_rejected_logits": -3.348952293395996, | |
| "debug/policy_rejected_logps": -133.3173828125, | |
| "debug/reference_chosen_logps": -154.70895385742188, | |
| "debug/reference_rejected_logps": -133.4116668701172, | |
| "epoch": 0.4594594594594595, | |
| "grad_norm": 3.6801385333166086, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.4201347827911377, | |
| "logits/rejected": -3.348952293395996, | |
| "logps/chosen": -154.7392578125, | |
| "logps/rejected": -133.3173828125, | |
| "loss": 0.4999, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.0003031063242815435, | |
| "rewards/margins": -0.0012458800338208675, | |
| "rewards/rejected": 0.0009427738841623068, | |
| "step": 17 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.2982780933380127, | |
| "debug/policy_chosen_logps": -146.35263061523438, | |
| "debug/policy_rejected_logits": -3.3202931880950928, | |
| "debug/policy_rejected_logps": -156.7283172607422, | |
| "debug/reference_chosen_logps": -146.34840393066406, | |
| "debug/reference_rejected_logps": -156.84378051757812, | |
| "epoch": 0.4864864864864865, | |
| "grad_norm": 3.5196394680278327, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.2982780933380127, | |
| "logits/rejected": -3.3202931880950928, | |
| "logps/chosen": -146.35263061523438, | |
| "logps/rejected": -156.7283172607422, | |
| "loss": 0.4974, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -4.220963455736637e-05, | |
| "rewards/margins": -0.0011968993349000812, | |
| "rewards/rejected": 0.0011546898167580366, | |
| "step": 18 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.3398616313934326, | |
| "debug/policy_chosen_logps": -131.99713134765625, | |
| "debug/policy_rejected_logits": -3.3604841232299805, | |
| "debug/policy_rejected_logps": -148.30227661132812, | |
| "debug/reference_chosen_logps": -131.88931274414062, | |
| "debug/reference_rejected_logps": -148.13345336914062, | |
| "epoch": 0.5135135135135135, | |
| "grad_norm": 3.609790844356815, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.3398616313934326, | |
| "logits/rejected": -3.3604841232299805, | |
| "logps/chosen": -131.99713134765625, | |
| "logps/rejected": -148.30227661132812, | |
| "loss": 0.4981, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.0010780429001897573, | |
| "rewards/margins": 0.0006100271129980683, | |
| "rewards/rejected": -0.0016880702460184693, | |
| "step": 19 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.2643423080444336, | |
| "debug/policy_chosen_logps": -134.54617309570312, | |
| "debug/policy_rejected_logits": -3.255173921585083, | |
| "debug/policy_rejected_logps": -124.35269165039062, | |
| "debug/reference_chosen_logps": -134.53549194335938, | |
| "debug/reference_rejected_logps": -123.25413513183594, | |
| "epoch": 0.5405405405405406, | |
| "grad_norm": 3.4308218789715035, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.2643423080444336, | |
| "logits/rejected": -3.255173921585083, | |
| "logps/chosen": -134.54617309570312, | |
| "logps/rejected": -124.35269165039062, | |
| "loss": 0.4969, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.00010671629570424557, | |
| "rewards/margins": 0.010878859087824821, | |
| "rewards/rejected": -0.010985574685037136, | |
| "step": 20 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.39021635055542, | |
| "debug/policy_chosen_logps": -145.20547485351562, | |
| "debug/policy_rejected_logits": -3.324305295944214, | |
| "debug/policy_rejected_logps": -140.68399047851562, | |
| "debug/reference_chosen_logps": -145.19155883789062, | |
| "debug/reference_rejected_logps": -139.9624481201172, | |
| "epoch": 0.5675675675675675, | |
| "grad_norm": 3.572312158938155, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.39021635055542, | |
| "logits/rejected": -3.324305295944214, | |
| "logps/chosen": -145.20547485351562, | |
| "logps/rejected": -140.68399047851562, | |
| "loss": 0.4975, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.00013918871991336346, | |
| "rewards/margins": 0.007076186593621969, | |
| "rewards/rejected": -0.007215375080704689, | |
| "step": 21 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.267886161804199, | |
| "debug/policy_chosen_logps": -160.2618865966797, | |
| "debug/policy_rejected_logits": -3.2591264247894287, | |
| "debug/policy_rejected_logps": -141.91876220703125, | |
| "debug/reference_chosen_logps": -159.80300903320312, | |
| "debug/reference_rejected_logps": -142.28695678710938, | |
| "epoch": 0.5945945945945946, | |
| "grad_norm": 3.81196901542863, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.267886161804199, | |
| "logits/rejected": -3.2591264247894287, | |
| "logps/chosen": -160.2618865966797, | |
| "logps/rejected": -141.91876220703125, | |
| "loss": 0.4995, | |
| "rewards/accuracies": 0.125, | |
| "rewards/chosen": -0.004588775336742401, | |
| "rewards/margins": -0.008270883932709694, | |
| "rewards/rejected": 0.0036821081303060055, | |
| "step": 22 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.3076066970825195, | |
| "debug/policy_chosen_logps": -147.66110229492188, | |
| "debug/policy_rejected_logits": -3.3076066970825195, | |
| "debug/policy_rejected_logps": -156.1669464111328, | |
| "debug/reference_chosen_logps": -147.6890869140625, | |
| "debug/reference_rejected_logps": -155.93359375, | |
| "epoch": 0.6216216216216216, | |
| "grad_norm": 3.584513321934443, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.3076066970825195, | |
| "logits/rejected": -3.3076066970825195, | |
| "logps/chosen": -147.66110229492188, | |
| "logps/rejected": -156.1669464111328, | |
| "loss": 0.4981, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.0002799700014293194, | |
| "rewards/margins": 0.002613373100757599, | |
| "rewards/rejected": -0.002333402633666992, | |
| "step": 23 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.3157832622528076, | |
| "debug/policy_chosen_logps": -145.93922424316406, | |
| "debug/policy_rejected_logits": -3.2806382179260254, | |
| "debug/policy_rejected_logps": -150.69491577148438, | |
| "debug/reference_chosen_logps": -146.20852661132812, | |
| "debug/reference_rejected_logps": -151.0212860107422, | |
| "epoch": 0.6486486486486487, | |
| "grad_norm": 3.4973421303279575, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.3157832622528076, | |
| "logits/rejected": -3.2806382179260254, | |
| "logps/chosen": -145.93922424316406, | |
| "logps/rejected": -150.69491577148438, | |
| "loss": 0.4979, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.0026929855812340975, | |
| "rewards/margins": -0.0005707358941435814, | |
| "rewards/rejected": 0.0032637212425470352, | |
| "step": 24 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.340818405151367, | |
| "debug/policy_chosen_logps": -145.6500244140625, | |
| "debug/policy_rejected_logits": -3.2608158588409424, | |
| "debug/policy_rejected_logps": -148.91769409179688, | |
| "debug/reference_chosen_logps": -145.28700256347656, | |
| "debug/reference_rejected_logps": -148.38320922851562, | |
| "epoch": 0.6756756756756757, | |
| "grad_norm": 3.493835228930821, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.340818405151367, | |
| "logits/rejected": -3.2608158588409424, | |
| "logps/chosen": -145.6500244140625, | |
| "logps/rejected": -148.91769409179688, | |
| "loss": 0.4997, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.003630065592005849, | |
| "rewards/margins": 0.0017146589234471321, | |
| "rewards/rejected": -0.005344724282622337, | |
| "step": 25 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.3708367347717285, | |
| "debug/policy_chosen_logps": -134.6257781982422, | |
| "debug/policy_rejected_logits": -3.2685859203338623, | |
| "debug/policy_rejected_logps": -164.87184143066406, | |
| "debug/reference_chosen_logps": -134.9357452392578, | |
| "debug/reference_rejected_logps": -164.06057739257812, | |
| "epoch": 0.7027027027027027, | |
| "grad_norm": 3.5262056978807492, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.3708367347717285, | |
| "logits/rejected": -3.2685859203338623, | |
| "logps/chosen": -134.6257781982422, | |
| "logps/rejected": -164.87184143066406, | |
| "loss": 0.4954, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.0030995942652225494, | |
| "rewards/margins": 0.011212329380214214, | |
| "rewards/rejected": -0.008112735114991665, | |
| "step": 26 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.276146650314331, | |
| "debug/policy_chosen_logps": -127.90383911132812, | |
| "debug/policy_rejected_logits": -3.2199714183807373, | |
| "debug/policy_rejected_logps": -148.08279418945312, | |
| "debug/reference_chosen_logps": -127.79987335205078, | |
| "debug/reference_rejected_logps": -148.602783203125, | |
| "epoch": 0.7297297297297297, | |
| "grad_norm": 3.5419489962048973, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.276146650314331, | |
| "logits/rejected": -3.2199714183807373, | |
| "logps/chosen": -127.90383911132812, | |
| "logps/rejected": -148.08279418945312, | |
| "loss": 0.4972, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": -0.0010396192083135247, | |
| "rewards/margins": -0.006239423528313637, | |
| "rewards/rejected": 0.005199803970754147, | |
| "step": 27 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.3671457767486572, | |
| "debug/policy_chosen_logps": -142.98072814941406, | |
| "debug/policy_rejected_logits": -3.313788652420044, | |
| "debug/policy_rejected_logps": -159.15054321289062, | |
| "debug/reference_chosen_logps": -143.54417419433594, | |
| "debug/reference_rejected_logps": -159.19894409179688, | |
| "epoch": 0.7567567567567568, | |
| "grad_norm": 3.6334386387375948, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.3671457767486572, | |
| "logits/rejected": -3.313788652420044, | |
| "logps/chosen": -142.98072814941406, | |
| "logps/rejected": -159.15054321289062, | |
| "loss": 0.4957, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.005634431727230549, | |
| "rewards/margins": 0.005150537472218275, | |
| "rewards/rejected": 0.0004838943714275956, | |
| "step": 28 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.451542377471924, | |
| "debug/policy_chosen_logps": -144.62733459472656, | |
| "debug/policy_rejected_logits": -3.500192403793335, | |
| "debug/policy_rejected_logps": -129.41226196289062, | |
| "debug/reference_chosen_logps": -144.27435302734375, | |
| "debug/reference_rejected_logps": -128.7982177734375, | |
| "epoch": 0.7837837837837838, | |
| "grad_norm": 3.525191396719062, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.451542377471924, | |
| "logits/rejected": -3.500192403793335, | |
| "logps/chosen": -144.62733459472656, | |
| "logps/rejected": -129.41226196289062, | |
| "loss": 0.4925, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.003529853653162718, | |
| "rewards/margins": 0.0026106643490493298, | |
| "rewards/rejected": -0.006140518002212048, | |
| "step": 29 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.4419167041778564, | |
| "debug/policy_chosen_logps": -128.94613647460938, | |
| "debug/policy_rejected_logits": -3.3551814556121826, | |
| "debug/policy_rejected_logps": -125.87129974365234, | |
| "debug/reference_chosen_logps": -129.1351318359375, | |
| "debug/reference_rejected_logps": -125.61131286621094, | |
| "epoch": 0.8108108108108109, | |
| "grad_norm": 3.4357174715493812, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.4419167041778564, | |
| "logits/rejected": -3.3551814556121826, | |
| "logps/chosen": -128.94613647460938, | |
| "logps/rejected": -125.87129974365234, | |
| "loss": 0.4956, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": 0.001890067826025188, | |
| "rewards/margins": 0.00448994617909193, | |
| "rewards/rejected": -0.0025998782366514206, | |
| "step": 30 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.464578628540039, | |
| "debug/policy_chosen_logps": -148.08114624023438, | |
| "debug/policy_rejected_logits": -3.42940354347229, | |
| "debug/policy_rejected_logps": -155.03309631347656, | |
| "debug/reference_chosen_logps": -147.857666015625, | |
| "debug/reference_rejected_logps": -153.7999267578125, | |
| "epoch": 0.8378378378378378, | |
| "grad_norm": 3.569444014406672, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.464578628540039, | |
| "logits/rejected": -3.42940354347229, | |
| "logps/chosen": -148.08114624023438, | |
| "logps/rejected": -155.03309631347656, | |
| "loss": 0.4994, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0022347732447087765, | |
| "rewards/margins": 0.010096902027726173, | |
| "rewards/rejected": -0.012331675738096237, | |
| "step": 31 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.2954604625701904, | |
| "debug/policy_chosen_logps": -146.03802490234375, | |
| "debug/policy_rejected_logits": -3.384243965148926, | |
| "debug/policy_rejected_logps": -145.24456787109375, | |
| "debug/reference_chosen_logps": -146.03688049316406, | |
| "debug/reference_rejected_logps": -144.96478271484375, | |
| "epoch": 0.8648648648648649, | |
| "grad_norm": 3.794862233420595, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.2954604625701904, | |
| "logits/rejected": -3.384243965148926, | |
| "logps/chosen": -146.03802490234375, | |
| "logps/rejected": -145.24456787109375, | |
| "loss": 0.4995, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -1.1482159607112408e-05, | |
| "rewards/margins": 0.002786235883831978, | |
| "rewards/rejected": -0.0027977179270237684, | |
| "step": 32 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.393167734146118, | |
| "debug/policy_chosen_logps": -135.54547119140625, | |
| "debug/policy_rejected_logits": -3.4107766151428223, | |
| "debug/policy_rejected_logps": -126.21063995361328, | |
| "debug/reference_chosen_logps": -135.9659423828125, | |
| "debug/reference_rejected_logps": -123.26311492919922, | |
| "epoch": 0.8918918918918919, | |
| "grad_norm": 3.7044090203550195, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.393167734146118, | |
| "logits/rejected": -3.4107766151428223, | |
| "logps/chosen": -135.54547119140625, | |
| "logps/rejected": -126.21063995361328, | |
| "loss": 0.4985, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.004204845521599054, | |
| "rewards/margins": 0.03368005529046059, | |
| "rewards/rejected": -0.02947521209716797, | |
| "step": 33 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.3235180377960205, | |
| "debug/policy_chosen_logps": -159.286865234375, | |
| "debug/policy_rejected_logits": -3.369253635406494, | |
| "debug/policy_rejected_logps": -165.31944274902344, | |
| "debug/reference_chosen_logps": -158.15670776367188, | |
| "debug/reference_rejected_logps": -164.94976806640625, | |
| "epoch": 0.918918918918919, | |
| "grad_norm": 3.550860802017907, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.3235180377960205, | |
| "logits/rejected": -3.369253635406494, | |
| "logps/chosen": -159.286865234375, | |
| "logps/rejected": -165.31944274902344, | |
| "loss": 0.4997, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": -0.011301536113023758, | |
| "rewards/margins": -0.007604693062603474, | |
| "rewards/rejected": -0.0036968423519283533, | |
| "step": 34 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.367034673690796, | |
| "debug/policy_chosen_logps": -129.437744140625, | |
| "debug/policy_rejected_logits": -3.396902561187744, | |
| "debug/policy_rejected_logps": -157.3302001953125, | |
| "debug/reference_chosen_logps": -129.46514892578125, | |
| "debug/reference_rejected_logps": -157.58755493164062, | |
| "epoch": 0.9459459459459459, | |
| "grad_norm": 3.6785421821803044, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.367034673690796, | |
| "logits/rejected": -3.396902561187744, | |
| "logps/chosen": -129.437744140625, | |
| "logps/rejected": -157.3302001953125, | |
| "loss": 0.498, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.00027409568428993225, | |
| "rewards/margins": -0.002299328101798892, | |
| "rewards/rejected": 0.002573424018919468, | |
| "step": 35 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.252328634262085, | |
| "debug/policy_chosen_logps": -139.81039428710938, | |
| "debug/policy_rejected_logits": -3.3797786235809326, | |
| "debug/policy_rejected_logps": -127.12653350830078, | |
| "debug/reference_chosen_logps": -140.35733032226562, | |
| "debug/reference_rejected_logps": -127.09129333496094, | |
| "epoch": 0.972972972972973, | |
| "grad_norm": 3.7364723193101446, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.252328634262085, | |
| "logits/rejected": -3.3797786235809326, | |
| "logps/chosen": -139.81039428710938, | |
| "logps/rejected": -127.12653350830078, | |
| "loss": 0.4951, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.005469245836138725, | |
| "rewards/margins": 0.0058215707540512085, | |
| "rewards/rejected": -0.0003523253835737705, | |
| "step": 36 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -3.166999101638794, | |
| "debug/policy_chosen_logps": -165.9640350341797, | |
| "debug/policy_rejected_logits": -3.1885428428649902, | |
| "debug/policy_rejected_logps": -135.82620239257812, | |
| "debug/reference_chosen_logps": -164.97256469726562, | |
| "debug/reference_rejected_logps": -135.76358032226562, | |
| "epoch": 1.0, | |
| "grad_norm": 3.5142818885997116, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -3.166999101638794, | |
| "logits/rejected": -3.1885428428649902, | |
| "logps/chosen": -165.9640350341797, | |
| "logps/rejected": -135.82620239257812, | |
| "loss": 0.481, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.009914855472743511, | |
| "rewards/margins": -0.009288596920669079, | |
| "rewards/rejected": -0.0006262586684897542, | |
| "step": 37 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 37, | |
| "total_flos": 0.0, | |
| "train_loss": 0.4980396749200048, | |
| "train_runtime": 139.9378, | |
| "train_samples_per_second": 16.757, | |
| "train_steps_per_second": 0.264 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 37, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |