Upload folder using huggingface_hub

README.md

@@ -6,28 +6,34 @@ tags:
 - lazymergekit
 - automerger
 base_model:
+- yam-peleg/Experiment26-7B
 - yam-peleg/Experiment28-7B
 ---
 
 # Experiment26Experiment28-7B
 
 Experiment26Experiment28-7B is an automated merge created by [Maxime Labonne](https://huggingface.co/mlabonne) using the following configuration.
+* [yam-peleg/Experiment26-7B](https://huggingface.co/yam-peleg/Experiment26-7B)
 * [yam-peleg/Experiment28-7B](https://huggingface.co/yam-peleg/Experiment28-7B)
 
 ## 🧩 Configuration
 
 ```yaml
-models:
-- model: rwitz/experiment26-truthy-iter-0
-  # No parameters necessary for base model
-- model: yam-peleg/Experiment28-7B
-  parameters:
-    density: 0.53
-    weight: 0.6
-merge_method: dare_ties
-base_model: rwitz/experiment26-truthy-iter-0
+slices:
+  - sources:
+      - model: yam-peleg/Experiment26-7B
+        layer_range: [0, 32]
+      - model: yam-peleg/Experiment28-7B
+        layer_range: [0, 32]
+merge_method: slerp
+base_model: yam-peleg/Experiment26-7B
 parameters:
-int8_mask: true
+  t:
+    - filter: self_attn
+      value: [0, 0.5, 0.3, 0.7, 1]
+    - filter: mlp
+      value: [1, 0.5, 0.7, 0.3, 0]
+    - value: 0.5
 dtype: bfloat16
 random_seed: 0
     ```

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "rwitz/experiment26-truthy-iter-0",
+  "_name_or_path": "yam-peleg/Experiment26-7B",
   "architectures": [
     "MistralForCausalLM"
   ],
@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.38.2",
+  "transformers_version": "4.39.0",
   "use_cache": true,
   "vocab_size": 32000
 }

mergekit_config.yml

@@ -1,15 +1,19 @@
 
-models:
-- model: rwitz/experiment26-truthy-iter-0
-  # No parameters necessary for base model
-- model: yam-peleg/Experiment28-7B
-  parameters:
-    density: 0.53
-    weight: 0.6
-merge_method: dare_ties
-base_model: rwitz/experiment26-truthy-iter-0
+slices:
+  - sources:
+      - model: yam-peleg/Experiment26-7B
+        layer_range: [0, 32]
+      - model: yam-peleg/Experiment28-7B
+        layer_range: [0, 32]
+merge_method: slerp
+base_model: yam-peleg/Experiment26-7B
 parameters:
-int8_mask: true
+  t:
+    - filter: self_attn
+      value: [0, 0.5, 0.3, 0.7, 1]
+    - filter: mlp
+      value: [1, 0.5, 0.7, 0.3, 0]
+    - value: 0.5
 dtype: bfloat16
 random_seed: 0
     

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b43aa5a9d908e87c45a6e8d0adf530f5bfb4490659cab6b0f4cec1340c8ec0f4
+oid sha256:7c2527ba2538f1e5f1a8593810370e6bf72f61f18976d1e7cdcf84e14a008754
 size 9942981696

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c241eecd76a700d76e563b52e90c89ccdd21eb9e5787416e8f7594d0031e2625
+oid sha256:6f8fd4b147717993bcf934c946412a23fc166f42a1192c4479e8b8d688b71388
 size 4540516344