trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 48,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.02,
      "grad_norm": 5.72966432783091,
      "learning_rate": 1e-07,
      "logits/chosen": -2.864940643310547,
      "logits/rejected": -2.8567800521850586,
      "logps/chosen": -154.45848083496094,
      "logps/rejected": -150.85018920898438,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.1,
      "grad_norm": 6.045391667595139,
      "learning_rate": 5e-07,
      "logits/chosen": -2.8156416416168213,
      "logits/rejected": -2.8174829483032227,
      "logps/chosen": -130.59176635742188,
      "logps/rejected": -143.8479766845703,
      "loss": 0.6931,
      "rewards/accuracies": 0.4140625,
      "rewards/chosen": 0.0010557841742411256,
      "rewards/margins": 0.0004505666147451848,
      "rewards/rejected": 0.0006052175885997713,
      "step": 5
    },
    {
      "epoch": 0.21,
      "grad_norm": 5.982972493722418,
      "learning_rate": 4.83504027183137e-07,
      "logits/chosen": -2.8159825801849365,
      "logits/rejected": -2.8173575401306152,
      "logps/chosen": -131.40081787109375,
      "logps/rejected": -144.49203491210938,
      "loss": 0.691,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.019764957949519157,
      "rewards/margins": 0.005402451381087303,
      "rewards/rejected": 0.014362509362399578,
      "step": 10
    },
    {
      "epoch": 0.31,
      "grad_norm": 5.340749871264835,
      "learning_rate": 4.3619304568594546e-07,
      "logits/chosen": -2.7662782669067383,
      "logits/rejected": -2.7731518745422363,
      "logps/chosen": -100.25220489501953,
      "logps/rejected": -114.28987121582031,
      "loss": 0.6863,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 0.05467454716563225,
      "rewards/margins": 0.011119803413748741,
      "rewards/rejected": 0.04355474188923836,
      "step": 15
    },
    {
      "epoch": 0.42,
      "grad_norm": 6.111662638062714,
      "learning_rate": 3.643105808261596e-07,
      "logits/chosen": -2.7775442600250244,
      "logits/rejected": -2.7860724925994873,
      "logps/chosen": -120.76275634765625,
      "logps/rejected": -118.23551940917969,
      "loss": 0.677,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.04566926881670952,
      "rewards/margins": 0.033490534871816635,
      "rewards/rejected": 0.01217872928828001,
      "step": 20
    },
    {
      "epoch": 0.52,
      "grad_norm": 6.802575221826661,
      "learning_rate": 2.7734280209446865e-07,
      "logits/chosen": -2.7327959537506104,
      "logits/rejected": -2.7430901527404785,
      "logps/chosen": -147.2881317138672,
      "logps/rejected": -157.8280792236328,
      "loss": 0.6643,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.014705592766404152,
      "rewards/margins": 0.05907569080591202,
      "rewards/rejected": -0.07378127425909042,
      "step": 25
    },
    {
      "epoch": 0.62,
      "grad_norm": 7.207930445306553,
      "learning_rate": 1.8676665440207977e-07,
      "logits/chosen": -2.767821788787842,
      "logits/rejected": -2.763593912124634,
      "logps/chosen": -127.16889953613281,
      "logps/rejected": -139.9912109375,
      "loss": 0.6661,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.009180300869047642,
      "rewards/margins": 0.05278443172574043,
      "rewards/rejected": -0.0619647316634655,
      "step": 30
    },
    {
      "epoch": 0.73,
      "grad_norm": 7.878619159771562,
      "learning_rate": 1.0453527111051183e-07,
      "logits/chosen": -2.766216993331909,
      "logits/rejected": -2.7657628059387207,
      "logps/chosen": -123.0304946899414,
      "logps/rejected": -140.19467163085938,
      "loss": 0.6557,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.010604464448988438,
      "rewards/margins": 0.07750671356916428,
      "rewards/rejected": -0.08811117708683014,
      "step": 35
    },
    {
      "epoch": 0.83,
      "grad_norm": 8.064504462502212,
      "learning_rate": 4.1500545527530544e-08,
      "logits/chosen": -2.741198778152466,
      "logits/rejected": -2.7608323097229004,
      "logps/chosen": -144.61236572265625,
      "logps/rejected": -162.44638061523438,
      "loss": 0.649,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0168136078864336,
      "rewards/margins": 0.12509755790233612,
      "rewards/rejected": -0.10828395932912827,
      "step": 40
    },
    {
      "epoch": 0.94,
      "grad_norm": 7.866795164549529,
      "learning_rate": 5.981030611018234e-09,
      "logits/chosen": -2.745185613632202,
      "logits/rejected": -2.7535369396209717,
      "logps/chosen": -119.3572998046875,
      "logps/rejected": -130.43502807617188,
      "loss": 0.6512,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.01881173998117447,
      "rewards/margins": 0.10116763412952423,
      "rewards/rejected": -0.08235590159893036,
      "step": 45
    },
    {
      "epoch": 1.0,
      "step": 48,
      "total_flos": 0.0,
      "train_loss": 0.6684800560275713,
      "train_runtime": 1303.1718,
      "train_samples_per_second": 4.691,
      "train_steps_per_second": 0.037
    }
  ],
  "logging_steps": 5,
  "max_steps": 48,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}