end

checkpoints/checkpoint_epoch=2499.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1846fa7198ed55388dcf730520e72a98f9b9483056f92c8964d558c0f88ed9de
+size 218842306

checkpoints/checkpoint_epoch=2599.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:673384e1021c0005a3345ee17c6c14239d4453c7c940db013f319aad6aeadb13
+size 218842306

checkpoints/checkpoint_epoch=2699.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca43ec07f833b767266fe851928f7c1ef1628ad0cac426792bfe329aec4c826c
+size 218842306

checkpoints/checkpoint_epoch=2799.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2ed31508937a591157d1dd6889283068875abb2cee50b46225e4de155834a45
+size 218842306

checkpoints/checkpoint_epoch=2899.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37de60f9729a3dff88b541339e9ad211b9145b5aeef0f0467e882dd875cd3fd2
+size 218842306

checkpoints/checkpoint_epoch=2999.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e0cc706f2c29a483e4910dfe3f3657f8133956c651a21593a52bb1d59cdfb64
+size 218842306

checkpoints/checkpoint_epoch=3099.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eca36c56b5f11e944cdaef9edbfe818a05820dd6193d4fa69d54e7fe5088fc7d
+size 218842306

checkpoints/checkpoint_epoch=3199.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:500c384d4c23cc412030e4140cb3fecce934850493abf7c361c06d45a0b48fdc
+size 218842306

checkpoints/checkpoint_epoch=3299.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffc03785b0dd0607f4a6e2565869e87466571d84ced1ee5d363ed8e414f24080
+size 218842306

checkpoints/checkpoint_epoch=3399.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd4d59e487b7fd27149d6a19cec7b7611c7a0de940e15b51c0f8c3e3b20a526
+size 218842306

checkpoints/last.ckpt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2edf02d21f458bee897bd2a15cdf5ee019372a83ded8bb1825796bdec82913e3
-size 218839561
+oid sha256:5182282e82fcbdbb45e5c04fa38b164ad1a81cfc1094cd2dbaf7cb3f1bd895c3
+size 218842306

tensorboard/version_0/events.out.tfevents.1729438040.842e86ef2042.2233.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e3dbe5ee7e2f714bedfedfc3813642043e717e2b2fe741b0f72a05bb6526148
-size 89670577
+oid sha256:296e938933539883d4f8c288af09ddfedaaef4435d1dcb125ffcf93357cae1f8
+size 1066585206

train.log

@@ -11,3 +11,43 @@
 [2024-10-20 15:27:20,323][__main__][INFO] - Instantiating trainer <lightning.pytorch.trainer.Trainer>
 [2024-10-20 15:27:20,367][__main__][INFO] - Logging hyperparameters!
 [2024-10-20 15:27:20,422][__main__][INFO] - Starting training!
+[2024-10-22 01:31:06,012][matcha.utils.utils][ERROR] - 
+Traceback (most recent call last):
+  File "/workspace/Matcha-TTS/matcha/utils/utils.py", line 77, in wrap
+    metric_dict, object_dict = task_func(cfg=cfg)
+  File "/workspace/Matcha-TTS/matcha/train.py", line 79, in train
+    trainer.fit(model=model, datamodule=datamodule, ckpt_path=cfg.get("ckpt_path"))
+  File "/opt/conda/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 538, in fit
+    call._call_and_handle_interrupt(
+  File "/opt/conda/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 47, in _call_and_handle_interrupt
+    return trainer_fn(*args, **kwargs)
+  File "/opt/conda/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 574, in _fit_impl
+    self._run(model, ckpt_path=ckpt_path)
+  File "/opt/conda/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 981, in _run
+    results = self._run_stage()
+  File "/opt/conda/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 1025, in _run_stage
+    self.fit_loop.run()
+  File "/opt/conda/lib/python3.10/site-packages/lightning/pytorch/loops/fit_loop.py", line 206, in run
+    self.on_advance_end()
+  File "/opt/conda/lib/python3.10/site-packages/lightning/pytorch/loops/fit_loop.py", line 378, in on_advance_end
+    call._call_callback_hooks(trainer, "on_train_epoch_end", monitoring_callbacks=True)
+  File "/opt/conda/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 218, in _call_callback_hooks
+    fn(trainer, trainer.lightning_module, *args, **kwargs)
+  File "/opt/conda/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py", line 326, in on_train_epoch_end
+    self._save_last_checkpoint(trainer, monitor_candidates)
+  File "/opt/conda/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py", line 696, in _save_last_checkpoint
+    self._save_checkpoint(trainer, filepath)
+  File "/opt/conda/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py", line 390, in _save_checkpoint
+    trainer.save_checkpoint(filepath, self.save_weights_only)
+  File "/opt/conda/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 1365, in save_checkpoint
+    self.strategy.save_checkpoint(checkpoint, filepath, storage_options=storage_options)
+  File "/opt/conda/lib/python3.10/site-packages/lightning/pytorch/strategies/strategy.py", line 490, in save_checkpoint
+    self.checkpoint_io.save_checkpoint(checkpoint, filepath, storage_options=storage_options)
+  File "/opt/conda/lib/python3.10/site-packages/lightning/fabric/plugins/io/torch_io.py", line 58, in save_checkpoint
+    _atomic_save(checkpoint, path)
+  File "/opt/conda/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py", line 90, in _atomic_save
+    f.write(bytesbuffer.getvalue())
+  File "/opt/conda/lib/python3.10/site-packages/fsspec/implementations/local.py", line 373, in write
+    return self.f.write(*args, **kwargs)
+OSError: [Errno 28] No space left on device
+[2024-10-22 01:31:06,020][matcha.utils.utils][INFO] - Output dir: /workspace/Matcha-TTS/logs/train/darkman/runs/2024-10-20_15-27-18