Upload folder using huggingface_hub

README.md

@@ -17,7 +17,7 @@ This is a merge of pre-trained language models created using [mergekit](https://
 ## Merge Details
 ### Merge Method
 
-This model was merged using the [Model Stock](https://arxiv.org/abs/2403.19522) merge method using [Undi95/Meta-Llama-3-8B-hf](https://huggingface.co/Undi95/Meta-Llama-3-8B-hf) as a base.
+This model was merged using the [TIES](https://arxiv.org/abs/2306.01708) merge method using [Undi95/Meta-Llama-3-8B-hf](https://huggingface.co/Undi95/Meta-Llama-3-8B-hf) as a base.
 
 ### Models Merged
 
@@ -31,24 +31,37 @@ The following models were included in the merge:
 The following YAML configuration was used to produce this model:
 
 ```yaml
+# Mergekit Configuration for Model Merge
+
+# Base model (primary reference model)
+base_model: Undi95/Meta-Llama-3-8B-hf
+
+# Merge method (using TIES for intelligent merging)
+merge_method: ties
+
+# Specific model configurations
 models:
   - model: Sao10K/L3-8B-Stheno-v3.2
     parameters:
       density: 0.4
-      weight: 0.4
+      weight: 0.25
+
   - model: ArliAI/Llama-3.1-8B-ArliAI-RPMax-v1.2
     parameters:
-      density: 0.4
-      weight: 0.4
+      density: 0.5
+      weight: 0.35
+
   - model: O1-OPEN/OpenO1-LLama-8B-v0.1
     parameters:
-      density: 0.2
-      weight: 0.2
+      density: 0.3
+      weight: 0.4
 
-merge_method: model_stock
-base_model: Undi95/Meta-Llama-3-8B-hf
+# Merge parameters
 parameters:
-  normalize: false
+  normalize: true
   int8_mask: true
-dtype: float16
+  dtype: 16  # Explicitly using 16-bit float representation
+
+# Tokenizer source (use base model's tokenizer)
+tokenizer_source: base
 ```

config.json

@@ -23,7 +23,7 @@
   "rope_scaling": null,
   "rope_theta": 500000.0,
   "tie_word_embeddings": false,
-  "torch_dtype": "float16",
+  "torch_dtype": "bfloat16",
   "transformers_version": "4.46.2",
   "use_cache": true,
   "vocab_size": 128256

mergekit_config.yml

@@ -1,20 +1,33 @@
+# Mergekit Configuration for Model Merge
+
+# Base model (primary reference model)
+base_model: Undi95/Meta-Llama-3-8B-hf
+
+# Merge method (using TIES for intelligent merging)
+merge_method: ties
+
+# Specific model configurations
 models:
   - model: Sao10K/L3-8B-Stheno-v3.2
     parameters:
       density: 0.4
-      weight: 0.4
+      weight: 0.25
+
   - model: ArliAI/Llama-3.1-8B-ArliAI-RPMax-v1.2
     parameters:
-      density: 0.4
-      weight: 0.4
+      density: 0.5
+      weight: 0.35
+
   - model: O1-OPEN/OpenO1-LLama-8B-v0.1
     parameters:
-      density: 0.2
-      weight: 0.2
+      density: 0.3
+      weight: 0.4
 
-merge_method: model_stock
-base_model: Undi95/Meta-Llama-3-8B-hf
+# Merge parameters
 parameters:
-  normalize: false
+  normalize: true
   int8_mask: true
-dtype: float16
+  dtype: 16  # Explicitly using 16-bit float representation
+
+# Tokenizer source (use base model's tokenizer)
+tokenizer_source: base

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3d0c0e202e6ac9ac4f61a97934be456a30c80e378631af5119492ffa62f7fdf6
-size 4953586328
+oid sha256:c687a0e8f1762a157cedf2071db97c0a531ae58b52416c428bc91f91a04cc740
+size 4953586384

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f82a682e97bf18a9d7b941d0e0e132f6c050ad390d6102a7f243f528d17b2eae
-size 4999819232
+oid sha256:0c11929dc607753260cde22662210843155f65e111c9f334709677da609636c0
+size 4999819336

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:10187beb3be86c67a0a215d398dc8455040ed139206b668c32b6a7c53f7468d9
-size 4915916048
+oid sha256:f37dae92a0436e8f5f12296fcacf31f590e415da7d9a59d89a1019e5e203931c
+size 4915916144

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:50b4b0d0a14a8c69ff90ecd19a1f58595c08528fee5e78dad7ab4669d62c7567
-size 1191234448
+oid sha256:45c02fbc450baf7a3cd00af92c0ec53dc5a655704310a9bc63f5ff1c2ac78271
+size 1191234472