zzz99's picture
Training in progress, epoch 1
ca0cb2e verified
{
"os": "Linux-4.14.336-253.554.amzn2.x86_64-x86_64-with-glibc2.35",
"python": "3.10.13",
"heartbeatAt": "2024-02-08T18:00:21.239172",
"startedAt": "2024-02-08T18:00:20.760995",
"docker": null,
"cuda": null,
"args": [],
"state": "running",
"program": "/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local.py",
"codePathLocal": null,
"host": "default",
"username": "sagemaker-user",
"executable": "/opt/conda/bin/python3",
"cpu_count": 96,
"cpu_count_logical": 192,
"cpu_freq": {
"current": 3011.898104166666,
"min": 0.0,
"max": 0.0
},
"cpu_freq_per_core": [
{
"current": 2708.532,
"min": 0.0,
"max": 0.0
},
{
"current": 2549.917,
"min": 0.0,
"max": 0.0
},
{
"current": 2649.342,
"min": 0.0,
"max": 0.0
},
{
"current": 2617.578,
"min": 0.0,
"max": 0.0
},
{
"current": 2514.64,
"min": 0.0,
"max": 0.0
},
{
"current": 3300.018,
"min": 0.0,
"max": 0.0
},
{
"current": 2514.814,
"min": 0.0,
"max": 0.0
},
{
"current": 2823.715,
"min": 0.0,
"max": 0.0
},
{
"current": 2444.923,
"min": 0.0,
"max": 0.0
},
{
"current": 2450.425,
"min": 0.0,
"max": 0.0
},
{
"current": 2405.045,
"min": 0.0,
"max": 0.0
},
{
"current": 3300.611,
"min": 0.0,
"max": 0.0
},
{
"current": 2521.7,
"min": 0.0,
"max": 0.0
},
{
"current": 2536.269,
"min": 0.0,
"max": 0.0
},
{
"current": 2454.76,
"min": 0.0,
"max": 0.0
},
{
"current": 2452.125,
"min": 0.0,
"max": 0.0
},
{
"current": 2578.243,
"min": 0.0,
"max": 0.0
},
{
"current": 2588.493,
"min": 0.0,
"max": 0.0
},
{
"current": 2597.93,
"min": 0.0,
"max": 0.0
},
{
"current": 2362.191,
"min": 0.0,
"max": 0.0
},
{
"current": 2603.048,
"min": 0.0,
"max": 0.0
},
{
"current": 1726.583,
"min": 0.0,
"max": 0.0
},
{
"current": 2010.457,
"min": 0.0,
"max": 0.0
},
{
"current": 1983.584,
"min": 0.0,
"max": 0.0
},
{
"current": 2585.543,
"min": 0.0,
"max": 0.0
},
{
"current": 2448.628,
"min": 0.0,
"max": 0.0
},
{
"current": 2473.293,
"min": 0.0,
"max": 0.0
},
{
"current": 2326.422,
"min": 0.0,
"max": 0.0
},
{
"current": 2469.471,
"min": 0.0,
"max": 0.0
},
{
"current": 1679.725,
"min": 0.0,
"max": 0.0
},
{
"current": 1707.891,
"min": 0.0,
"max": 0.0
},
{
"current": 2608.243,
"min": 0.0,
"max": 0.0
},
{
"current": 1825.706,
"min": 0.0,
"max": 0.0
},
{
"current": 1852.851,
"min": 0.0,
"max": 0.0
},
{
"current": 2741.597,
"min": 0.0,
"max": 0.0
},
{
"current": 2763.617,
"min": 0.0,
"max": 0.0
},
{
"current": 2668.973,
"min": 0.0,
"max": 0.0
},
{
"current": 1848.906,
"min": 0.0,
"max": 0.0
},
{
"current": 1859.662,
"min": 0.0,
"max": 0.0
},
{
"current": 1872.243,
"min": 0.0,
"max": 0.0
},
{
"current": 2750.258,
"min": 0.0,
"max": 0.0
},
{
"current": 2171.724,
"min": 0.0,
"max": 0.0
},
{
"current": 2673.366,
"min": 0.0,
"max": 0.0
},
{
"current": 2686.6,
"min": 0.0,
"max": 0.0
},
{
"current": 2254.628,
"min": 0.0,
"max": 0.0
},
{
"current": 1922.174,
"min": 0.0,
"max": 0.0
},
{
"current": 1919.649,
"min": 0.0,
"max": 0.0
},
{
"current": 1914.156,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.749,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.574,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.179,
"min": 0.0,
"max": 0.0
},
{
"current": 3300.775,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.087,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.329,
"min": 0.0,
"max": 0.0
},
{
"current": 2200.612,
"min": 0.0,
"max": 0.0
},
{
"current": 2744.304,
"min": 0.0,
"max": 0.0
},
{
"current": 3300.722,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.792,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.956,
"min": 0.0,
"max": 0.0
},
{
"current": 3301.117,
"min": 0.0,
"max": 0.0
},
{
"current": 2226.489,
"min": 0.0,
"max": 0.0
},
{
"current": 3070.859,
"min": 0.0,
"max": 0.0
},
{
"current": 1992.465,
"min": 0.0,
"max": 0.0
},
{
"current": 3048.294,
"min": 0.0,
"max": 0.0
},
{
"current": 2694.481,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.499,
"min": 0.0,
"max": 0.0
},
{
"current": 2794.422,
"min": 0.0,
"max": 0.0
},
{
"current": 3292.924,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.232,
"min": 0.0,
"max": 0.0
},
{
"current": 3297.378,
"min": 0.0,
"max": 0.0
},
{
"current": 3300.519,
"min": 0.0,
"max": 0.0
},
{
"current": 3300.987,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.537,
"min": 0.0,
"max": 0.0
},
{
"current": 2474.441,
"min": 0.0,
"max": 0.0
},
{
"current": 2795.47,
"min": 0.0,
"max": 0.0
},
{
"current": 2412.968,
"min": 0.0,
"max": 0.0
},
{
"current": 2550.34,
"min": 0.0,
"max": 0.0
},
{
"current": 3009.464,
"min": 0.0,
"max": 0.0
},
{
"current": 2578.135,
"min": 0.0,
"max": 0.0
},
{
"current": 2404.379,
"min": 0.0,
"max": 0.0
},
{
"current": 3020.541,
"min": 0.0,
"max": 0.0
},
{
"current": 3058.728,
"min": 0.0,
"max": 0.0
},
{
"current": 2989.512,
"min": 0.0,
"max": 0.0
},
{
"current": 3059.601,
"min": 0.0,
"max": 0.0
},
{
"current": 2714.737,
"min": 0.0,
"max": 0.0
},
{
"current": 2694.293,
"min": 0.0,
"max": 0.0
},
{
"current": 3261.941,
"min": 0.0,
"max": 0.0
},
{
"current": 2586.965,
"min": 0.0,
"max": 0.0
},
{
"current": 3294.605,
"min": 0.0,
"max": 0.0
},
{
"current": 3260.796,
"min": 0.0,
"max": 0.0
},
{
"current": 3295.156,
"min": 0.0,
"max": 0.0
},
{
"current": 3253.521,
"min": 0.0,
"max": 0.0
},
{
"current": 2832.733,
"min": 0.0,
"max": 0.0
},
{
"current": 3264.153,
"min": 0.0,
"max": 0.0
},
{
"current": 2767.538,
"min": 0.0,
"max": 0.0
},
{
"current": 3300.784,
"min": 0.0,
"max": 0.0
},
{
"current": 2342.793,
"min": 0.0,
"max": 0.0
},
{
"current": 2291.218,
"min": 0.0,
"max": 0.0
},
{
"current": 2343.318,
"min": 0.0,
"max": 0.0
},
{
"current": 2377.449,
"min": 0.0,
"max": 0.0
},
{
"current": 2182.822,
"min": 0.0,
"max": 0.0
},
{
"current": 3300.271,
"min": 0.0,
"max": 0.0
},
{
"current": 2180.405,
"min": 0.0,
"max": 0.0
},
{
"current": 2708.15,
"min": 0.0,
"max": 0.0
},
{
"current": 2217.841,
"min": 0.0,
"max": 0.0
},
{
"current": 2223.228,
"min": 0.0,
"max": 0.0
},
{
"current": 2332.486,
"min": 0.0,
"max": 0.0
},
{
"current": 2585.795,
"min": 0.0,
"max": 0.0
},
{
"current": 2332.572,
"min": 0.0,
"max": 0.0
},
{
"current": 2256.902,
"min": 0.0,
"max": 0.0
},
{
"current": 2334.544,
"min": 0.0,
"max": 0.0
},
{
"current": 2350.832,
"min": 0.0,
"max": 0.0
},
{
"current": 2424.323,
"min": 0.0,
"max": 0.0
},
{
"current": 2456.004,
"min": 0.0,
"max": 0.0
},
{
"current": 2457.45,
"min": 0.0,
"max": 0.0
},
{
"current": 2579.468,
"min": 0.0,
"max": 0.0
},
{
"current": 2458.733,
"min": 0.0,
"max": 0.0
},
{
"current": 1860.983,
"min": 0.0,
"max": 0.0
},
{
"current": 2198.204,
"min": 0.0,
"max": 0.0
},
{
"current": 2131.955,
"min": 0.0,
"max": 0.0
},
{
"current": 2416.616,
"min": 0.0,
"max": 0.0
},
{
"current": 2498.284,
"min": 0.0,
"max": 0.0
},
{
"current": 2409.271,
"min": 0.0,
"max": 0.0
},
{
"current": 2442.917,
"min": 0.0,
"max": 0.0
},
{
"current": 2387.494,
"min": 0.0,
"max": 0.0
},
{
"current": 1840.589,
"min": 0.0,
"max": 0.0
},
{
"current": 1851.316,
"min": 0.0,
"max": 0.0
},
{
"current": 2572.071,
"min": 0.0,
"max": 0.0
},
{
"current": 1846.514,
"min": 0.0,
"max": 0.0
},
{
"current": 1838.129,
"min": 0.0,
"max": 0.0
},
{
"current": 2458.548,
"min": 0.0,
"max": 0.0
},
{
"current": 2468.97,
"min": 0.0,
"max": 0.0
},
{
"current": 2573.521,
"min": 0.0,
"max": 0.0
},
{
"current": 1854.263,
"min": 0.0,
"max": 0.0
},
{
"current": 1852.806,
"min": 0.0,
"max": 0.0
},
{
"current": 1826.095,
"min": 0.0,
"max": 0.0
},
{
"current": 2445.646,
"min": 0.0,
"max": 0.0
},
{
"current": 2189.591,
"min": 0.0,
"max": 0.0
},
{
"current": 2264.349,
"min": 0.0,
"max": 0.0
},
{
"current": 2377.441,
"min": 0.0,
"max": 0.0
},
{
"current": 2012.723,
"min": 0.0,
"max": 0.0
},
{
"current": 1906.662,
"min": 0.0,
"max": 0.0
},
{
"current": 2124.071,
"min": 0.0,
"max": 0.0
},
{
"current": 1895.988,
"min": 0.0,
"max": 0.0
},
{
"current": 3300.029,
"min": 0.0,
"max": 0.0
},
{
"current": 2525.577,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.997,
"min": 0.0,
"max": 0.0
},
{
"current": 2670.847,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.699,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.175,
"min": 0.0,
"max": 0.0
},
{
"current": 2197.86,
"min": 0.0,
"max": 0.0
},
{
"current": 2528.763,
"min": 0.0,
"max": 0.0
},
{
"current": 3197.634,
"min": 0.0,
"max": 0.0
},
{
"current": 3211.437,
"min": 0.0,
"max": 0.0
},
{
"current": 3251.423,
"min": 0.0,
"max": 0.0
},
{
"current": 2959.085,
"min": 0.0,
"max": 0.0
},
{
"current": 2065.145,
"min": 0.0,
"max": 0.0
},
{
"current": 2974.061,
"min": 0.0,
"max": 0.0
},
{
"current": 2068.626,
"min": 0.0,
"max": 0.0
},
{
"current": 2986.901,
"min": 0.0,
"max": 0.0
},
{
"current": 2340.617,
"min": 0.0,
"max": 0.0
},
{
"current": 3032.214,
"min": 0.0,
"max": 0.0
},
{
"current": 2431.142,
"min": 0.0,
"max": 0.0
},
{
"current": 3302.396,
"min": 0.0,
"max": 0.0
},
{
"current": 2944.793,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.677,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.69,
"min": 0.0,
"max": 0.0
},
{
"current": 2920.67,
"min": 0.0,
"max": 0.0
},
{
"current": 3090.867,
"min": 0.0,
"max": 0.0
},
{
"current": 2503.478,
"min": 0.0,
"max": 0.0
},
{
"current": 2473.681,
"min": 0.0,
"max": 0.0
},
{
"current": 2485.275,
"min": 0.0,
"max": 0.0
},
{
"current": 2488.179,
"min": 0.0,
"max": 0.0
},
{
"current": 2990.001,
"min": 0.0,
"max": 0.0
},
{
"current": 2452.206,
"min": 0.0,
"max": 0.0
},
{
"current": 2729.116,
"min": 0.0,
"max": 0.0
},
{
"current": 3001.225,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.389,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.765,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.125,
"min": 0.0,
"max": 0.0
},
{
"current": 2002.363,
"min": 0.0,
"max": 0.0
},
{
"current": 2302.227,
"min": 0.0,
"max": 0.0
},
{
"current": 3132.468,
"min": 0.0,
"max": 0.0
},
{
"current": 2770.449,
"min": 0.0,
"max": 0.0
},
{
"current": 3288.126,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.022,
"min": 0.0,
"max": 0.0
},
{
"current": 3298.691,
"min": 0.0,
"max": 0.0
},
{
"current": 3297.894,
"min": 0.0,
"max": 0.0
},
{
"current": 2828.863,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.914,
"min": 0.0,
"max": 0.0
},
{
"current": 2721.595,
"min": 0.0,
"max": 0.0
},
{
"current": 3299.427,
"min": 0.0,
"max": 0.0
}
],
"disk": {
"/": {
"total": 32.0,
"used": 0.01256561279296875
}
},
"gpu": "NVIDIA A10G",
"gpu_count": 8,
"gpu_devices": [
{
"name": "NVIDIA A10G",
"memory_total": 24146608128
},
{
"name": "NVIDIA A10G",
"memory_total": 24146608128
},
{
"name": "NVIDIA A10G",
"memory_total": 24146608128
},
{
"name": "NVIDIA A10G",
"memory_total": 24146608128
},
{
"name": "NVIDIA A10G",
"memory_total": 24146608128
},
{
"name": "NVIDIA A10G",
"memory_total": 24146608128
},
{
"name": "NVIDIA A10G",
"memory_total": 24146608128
},
{
"name": "NVIDIA A10G",
"memory_total": 24146608128
}
],
"memory": {
"total": 747.9597625732422
}
}