QLoRA applied #3

README.md

@@ -33,13 +33,17 @@ More information needed
 The following hyperparameters were used during training:
 - learning_rate: 0.0002
 - train_batch_size: 2
-- eval_batch_size: 1
+- eval_batch_size: 8
 - seed: 42
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: constant
 - lr_scheduler_warmup_ratio: 0.03
 - training_steps: 500
 
+### Training results
+
+
+
 ### Framework versions
 
 - Transformers 4.34.0

adapter_config.json

@@ -17,9 +17,9 @@
   "revision": null,
   "target_modules": [
     "dense",
-    "query_key_value",
+    "dense_h_to_4h",
     "dense_4h_to_h",
-    "dense_h_to_4h"
+    "query_key_value"
   ],
   "task_type": "QUESTION_ANSWERING"
 }

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9ea1eb39d8ceff97a117811fe16774356e662db177421214593ba8696ca1577e
-size 261227285
+oid sha256:2adc0882d2fac6bc73446eb3ee2aab650549ba5f8cdff5abf642b8a61d878c35
+size 261226517

tokenizer.json

@@ -1,11 +1,6 @@
 {
   "version": "1.0",
-  "truncation": {
-    "direction": "Right",
-    "max_length": 384,
-    "strategy": "OnlySecond",
-    "stride": 128
-  },
+  "truncation": null,
   "padding": null,
   "added_tokens": [
     {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc96c2a3184d35d7403313432c71e978ee6768981e0bfb651e41e3e73a264c21
+oid sha256:d140dc39254311114541b93880aa4c7d50d5096304c401d5aca9d2967ea1483a
 size 4091