Upload folder using huggingface_hub

README.md

@@ -0,0 +1,1050 @@
+---
+base_model:
+- mistralai/Mistral-7B-v0.3
+- meta-math/MetaMath-Mistral-7B
+- uukuguy/speechless-zephyr-code-functionary-7b
+library_name: transformers
+tags:
+- mergekit
+- merge
+
+---
+# Yosegi-2
+
+This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
+
+## Merge Details
+### Merge Method
+
+This model was merged using the [TIES](https://arxiv.org/abs/2306.01708) merge method using ./Yosegi-0601 as a base.
+
+### Models Merged
+
+The following models were included in the merge:
+* [mistralai/Mistral-7B-v0.3](https://huggingface.co/mistralai/Mistral-7B-v0.3)
+* [meta-math/MetaMath-Mistral-7B](https://huggingface.co/meta-math/MetaMath-Mistral-7B)
+* ./Ninja-v1-RP-expressive-v2-LoRA
+* [uukuguy/speechless-zephyr-code-functionary-7b](https://huggingface.co/uukuguy/speechless-zephyr-code-functionary-7b)
+
+### Configuration
+
+The following YAML configuration was used to produce this model:
+
+```yaml
+base_model: ./Yosegi-0601
+dtype: bfloat16
+merge_method: ties
+parameters:
+  int8_mask: 1.0
+  normalize: 0.0
+slices:
+- sources:
+  - layer_range: [0, 2]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.9895701336232673
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.5057237984975562
+      - filter: mlp
+        value: 0.36247235528151495
+      - value: 0.0076810835717692014
+  - layer_range: [0, 2]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 0.8239779346577963
+      weight:
+      - filter: self_attn
+        value: 0.27499287617186813
+      - filter: mlp
+        value: 0.10579959634086915
+      - value: 0.14502290477239704
+  - layer_range: [0, 2]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.9654867628269999
+      - value: 0.9584724004158125
+      weight:
+      - filter: self_attn
+        value: 0.059719404899177556
+      - filter: mlp
+        value: 0.1299695859327612
+      - value: 0.18821871354400985
+  - layer_range: [0, 2]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.9322987005873715
+      - filter: mlp
+        value: 0.8119693860979944
+      - value: 0.7800996941956229
+      weight:
+      - filter: self_attn
+        value: 0.14989333734000856
+      - filter: mlp
+        value: 0.20525182711733667
+      - value: 0.0743540962371737
+  - layer_range: [0, 2]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [2, 4]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.6361163471256639
+      - filter: mlp
+        value: 0.9983948965135213
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.2433049522842103
+      - filter: mlp
+        value: 0.11537153133586801
+      - value: 0.11236945502439658
+  - layer_range: [2, 4]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.13087986863180992
+      - filter: mlp
+        value: 0.05060452788200992
+      - value: 0.029882383396623725
+  - layer_range: [2, 4]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.9938109261305853
+      - filter: mlp
+        value: 0.709432587913349
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.15343343058938377
+      - filter: mlp
+        value: 0.4105917936868785
+      - value: 0.6078632204623161
+  - layer_range: [2, 4]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 0.9634269234020544
+      weight:
+      - filter: self_attn
+        value: 0.03750763360681478
+      - filter: mlp
+        value: 0.29089122858987404
+      - value: 0.3408085857388722
+  - layer_range: [2, 4]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [4, 6]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.8057109303418598
+      - filter: mlp
+        value: 0.9954520808628292
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.02598285706585618
+      - filter: mlp
+        value: 0.06661629726622949
+      - value: 0.1285191000066376
+  - layer_range: [4, 6]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.9112825916608848
+      - filter: mlp
+        value: 0.9322557507910056
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.18823564379986454
+      - filter: mlp
+        value: 0.4552822441636322
+      - value: 0.5120525709221785
+  - layer_range: [4, 6]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.9869122169774399
+      - filter: mlp
+        value: 1.0
+      - value: 0.9751291459565757
+      weight:
+      - filter: self_attn
+        value: 0.00493134813843582
+      - filter: mlp
+        value: 0.3008979965262413
+      - value: 0.2528466849993097
+  - layer_range: [4, 6]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.8956512783019246
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.4197408619693966
+      - filter: mlp
+        value: 0.1448902874618845
+      - value: 0.5196932662212128
+  - layer_range: [4, 6]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [6, 8]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.05321377226808306
+      - filter: mlp
+        value: 0.0482589904702303
+      - value: 0.433407006546336
+  - layer_range: [6, 8]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.8300482882633113
+      - filter: mlp
+        value: 0.8951636861593875
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.35952608658046414
+      - filter: mlp
+        value: 0.17385333183950857
+      - value: 0.6366514725970246
+  - layer_range: [6, 8]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.7848308077099464
+      - filter: mlp
+        value: 0.869549457974157
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.12433943050311849
+      - filter: mlp
+        value: 0.3065832590226165
+      - value: 0.33138948726149514
+  - layer_range: [6, 8]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.11885967308786714
+      - filter: mlp
+        value: 0.29125668567121127
+      - value: 0.19251901269486088
+  - layer_range: [6, 8]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [8, 10]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.9429625513013793
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.4085396076816443
+      - filter: mlp
+        value: 0.038473657720644636
+      - value: 0.35014489493395495
+  - layer_range: [8, 10]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.26957216810533163
+      - filter: mlp
+        value: 0.2393300696241166
+      - value: 0.4735322427351712
+  - layer_range: [8, 10]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.8594757954447017
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.26101395702355007
+      - filter: mlp
+        value: 0.3147672140145126
+      - value: 0.11658182776184756
+  - layer_range: [8, 10]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.6948062341711919
+      - value: 0.9312401427737346
+      weight:
+      - filter: self_attn
+        value: 0.1987774487170517
+      - filter: mlp
+        value: 0.5628384475763534
+      - value: 0.2765378221890683
+  - layer_range: [8, 10]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [10, 12]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.8230035654228713
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.1741591536775035
+      - filter: mlp
+        value: 0.30563583223301516
+      - value: 0.2060419023239155
+  - layer_range: [10, 12]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.9991063013557119
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.1470996125766866
+      - filter: mlp
+        value: 0.06646481892400827
+      - value: 0.2645489609472036
+  - layer_range: [10, 12]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.6812899560643833
+      - filter: mlp
+        value: 0.9083104648631823
+      - value: 0.9730062683598184
+      weight:
+      - filter: self_attn
+        value: 0.14278507832578724
+      - filter: mlp
+        value: 0.3475945971407978
+      - value: 0.40266546962595284
+  - layer_range: [10, 12]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.7047231879232164
+      - filter: mlp
+        value: 0.9148432633716144
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.15341559366405985
+      - filter: mlp
+        value: 0.20047704006010095
+      - value: 0.17364445581398172
+  - layer_range: [10, 12]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [12, 14]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.6974090973508299
+      - filter: mlp
+        value: 1.0
+      - value: 0.9553573565285324
+      weight:
+      - filter: self_attn
+        value: 0.03614401712451334
+      - filter: mlp
+        value: 0.1287785039219736
+      - value: 0.3780545754310749
+  - layer_range: [12, 14]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.7857328784783159
+      - filter: mlp
+        value: 1.0
+      - value: 0.6631303877423032
+      weight:
+      - filter: self_attn
+        value: 0.21728574423632604
+      - filter: mlp
+        value: 0.22813107248290188
+      - value: 0.1435266378249425
+  - layer_range: [12, 14]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.7579910864422339
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.21526786827735228
+      - filter: mlp
+        value: 0.19769619474642783
+      - value: 0.49420458585638627
+  - layer_range: [12, 14]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.8379590665264793
+      - filter: mlp
+        value: 1.0
+      - value: 0.6778673543559375
+      weight:
+      - filter: self_attn
+        value: 0.060679858649663874
+      - filter: mlp
+        value: 0.17248738428562518
+      - value: 0.05145640258269078
+  - layer_range: [12, 14]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [14, 16]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.8193296716327286
+      - value: 0.709644132681917
+      weight:
+      - filter: self_attn
+        value: 0.09821428505487592
+      - filter: mlp
+        value: 0.0039875777021436964
+      - value: 0.27550746634944184
+  - layer_range: [14, 16]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.9420135087156387
+      - filter: mlp
+        value: 1.0
+      - value: 0.9478569230341948
+      weight:
+      - filter: self_attn
+        value: 0.32640822225239857
+      - filter: mlp
+        value: 0.28189746971019747
+      - value: 0.09777040841174603
+  - layer_range: [14, 16]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.9811539353914964
+      - filter: mlp
+        value: 1.0
+      - value: 0.9947034500579488
+      weight:
+      - filter: self_attn
+        value: 0.015308461456516246
+      - filter: mlp
+        value: 0.0018966958379955934
+      - value: 0.24275389952300747
+  - layer_range: [14, 16]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.9022355771447704
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.03331841447575224
+      - filter: mlp
+        value: 0.03561712850019841
+      - value: 0.16096143804589919
+  - layer_range: [14, 16]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [16, 18]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.8813466618200871
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.20435001101909528
+      - filter: mlp
+        value: 0.1516594727144469
+      - value: 0.2269819409999868
+  - layer_range: [16, 18]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 0.8113796412034742
+      weight:
+      - filter: self_attn
+        value: 0.23760349395229585
+      - filter: mlp
+        value: 0.1725436279774783
+      - value: 0.5818814139457673
+  - layer_range: [16, 18]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.9307369835995082
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.0673898519051937
+      - filter: mlp
+        value: 0.049368399457210624
+      - value: 0.2621269048339309
+  - layer_range: [16, 18]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.8219541044757637
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.21320061393511042
+      - filter: mlp
+        value: 0.09188781867337345
+      - value: 0.27266490524762327
+  - layer_range: [16, 18]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [18, 20]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.7993530327131696
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.20420262433348008
+      - filter: mlp
+        value: 0.43400570066910155
+      - value: 0.13720822682656159
+  - layer_range: [18, 20]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 0.7035563346885239
+      weight:
+      - filter: self_attn
+        value: 0.3313523263002212
+      - filter: mlp
+        value: 0.356035051194268
+      - value: 0.4742357680522683
+  - layer_range: [18, 20]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.2475654838180605
+      - filter: mlp
+        value: 0.35095371882044646
+      - value: 0.18536862919946695
+  - layer_range: [18, 20]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.02997204931537696
+      - filter: mlp
+        value: 0.4103581291392323
+      - value: 0.19313933251158066
+  - layer_range: [18, 20]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [20, 22]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 0.5321196166337413
+      weight:
+      - filter: self_attn
+        value: 0.17930537920958298
+      - filter: mlp
+        value: 0.07662274511683252
+      - value: 0.1354315278471591
+  - layer_range: [20, 22]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.3768803907042144
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.1592147705254305
+      - filter: mlp
+        value: 0.18410207999201075
+      - value: 0.4928015910047033
+  - layer_range: [20, 22]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.37897278298418885
+      - filter: mlp
+        value: 0.0952591073533606
+      - value: 0.03551732810121447
+  - layer_range: [20, 22]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.2682334102128691
+      - filter: mlp
+        value: 0.33485781481395227
+      - value: 0.3395139468281392
+  - layer_range: [20, 22]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [22, 24]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.8002588203446623
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.2549204541625693
+      - filter: mlp
+        value: 0.3722418477156178
+      - value: 0.2410463731352089
+  - layer_range: [22, 24]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.9220873255898425
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.487455295718532
+      - filter: mlp
+        value: 0.40022413917173594
+      - value: 0.17846009757502157
+  - layer_range: [22, 24]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.7696341317318985
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.011267799816515114
+      - filter: mlp
+        value: 0.5320959832591042
+      - value: 0.17095406531325266
+  - layer_range: [22, 24]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.556101646343872
+      - filter: mlp
+        value: 0.5470253909079791
+      - value: 0.13241555469863223
+  - layer_range: [22, 24]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [24, 26]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.8667033674916582
+      - filter: mlp
+        value: 1.0
+      - value: 0.9446091486920749
+      weight:
+      - filter: self_attn
+        value: 0.4134110775513897
+      - filter: mlp
+        value: 0.0181822765943834
+      - value: 0.22797659617038232
+  - layer_range: [24, 26]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.9839865829690491
+      - value: 0.8252981103449059
+      weight:
+      - filter: self_attn
+        value: 0.3310295320944009
+      - filter: mlp
+        value: 0.05341478458353629
+      - value: 0.3588847186159219
+  - layer_range: [24, 26]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.8834823812212265
+      - value: 0.8195593509048733
+      weight:
+      - filter: self_attn
+        value: 0.3778012590489552
+      - filter: mlp
+        value: 0.2553204906819882
+      - value: 0.23250565137970108
+  - layer_range: [24, 26]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.7731602497153744
+      - value: 0.8647152680789973
+      weight:
+      - filter: self_attn
+        value: 0.1101209698118704
+      - filter: mlp
+        value: 0.2399169741437055
+      - value: 0.32311925187355206
+  - layer_range: [24, 26]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [26, 28]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.9508674341172941
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.4312865186270921
+      - filter: mlp
+        value: 0.28336325917543326
+      - value: 0.051826325177477234
+  - layer_range: [26, 28]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.8945725432745376
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.03524133636598346
+      - filter: mlp
+        value: 0.21426126710725438
+      - value: 0.31724116335002545
+  - layer_range: [26, 28]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.7138130384877139
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.04890129864608137
+      - filter: mlp
+        value: 0.3324333287494201
+      - value: 0.11533647335498036
+  - layer_range: [26, 28]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 0.9200281327001997
+      weight:
+      - filter: self_attn
+        value: 0.300842776105564
+      - filter: mlp
+        value: 0.08363140003203932
+      - value: 0.2538677006866867
+  - layer_range: [26, 28]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [28, 30]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.7116000185808022
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.10977758983122704
+      - filter: mlp
+        value: 0.1839207861311269
+      - value: 0.5426174846632369
+  - layer_range: [28, 30]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.8412049419861911
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.3517232690814979
+      - filter: mlp
+        value: 0.11878679655495025
+      - value: 0.432611353923264
+  - layer_range: [28, 30]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.7196182744068202
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.29848623969081
+      - filter: mlp
+        value: 0.034661358236493495
+      - value: 0.3438376072572394
+  - layer_range: [28, 30]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.051511204430449285
+      - filter: mlp
+        value: 0.3617968383178797
+      - value: 0.2578690795635758
+  - layer_range: [28, 30]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [30, 32]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.7971002248466003
+      - filter: mlp
+        value: 0.8931695149333363
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.07401430804790136
+      - filter: mlp
+        value: 0.00696466997386886
+      - value: 0.08295038526296711
+  - layer_range: [30, 32]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.8158777337631619
+      - value: 0.8348784699583887
+      weight:
+      - filter: self_attn
+        value: 0.26799125918248423
+      - filter: mlp
+        value: 0.08176923813129498
+      - value: 0.030317330226146508
+  - layer_range: [30, 32]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.8188850632365792
+      - filter: mlp
+        value: 0.7463831519693573
+      - value: 0.6515317051533988
+      weight:
+      - filter: self_attn
+        value: 0.21122007850953434
+      - filter: mlp
+        value: 0.1463362342258229
+      - value: 0.09176704194956312
+  - layer_range: [30, 32]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.9313941807354906
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.1443680121177074
+      - filter: mlp
+        value: 0.08309606396368145
+      - value: 0.37059044424517035
+  - layer_range: [30, 32]
+    model: ./Yosegi-0601
+
+```

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "./Yosegi-0601",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.41.1",
+  "use_cache": true,
+  "vocab_size": 32000
+}

mergekit_config.yml

@@ -0,0 +1,1015 @@
+base_model: ./Yosegi-0601
+dtype: bfloat16
+merge_method: ties
+parameters:
+  int8_mask: 1.0
+  normalize: 0.0
+slices:
+- sources:
+  - layer_range: [0, 2]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.9895701336232673
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.5057237984975562
+      - filter: mlp
+        value: 0.36247235528151495
+      - value: 0.0076810835717692014
+  - layer_range: [0, 2]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 0.8239779346577963
+      weight:
+      - filter: self_attn
+        value: 0.27499287617186813
+      - filter: mlp
+        value: 0.10579959634086915
+      - value: 0.14502290477239704
+  - layer_range: [0, 2]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.9654867628269999
+      - value: 0.9584724004158125
+      weight:
+      - filter: self_attn
+        value: 0.059719404899177556
+      - filter: mlp
+        value: 0.1299695859327612
+      - value: 0.18821871354400985
+  - layer_range: [0, 2]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.9322987005873715
+      - filter: mlp
+        value: 0.8119693860979944
+      - value: 0.7800996941956229
+      weight:
+      - filter: self_attn
+        value: 0.14989333734000856
+      - filter: mlp
+        value: 0.20525182711733667
+      - value: 0.0743540962371737
+  - layer_range: [0, 2]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [2, 4]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.6361163471256639
+      - filter: mlp
+        value: 0.9983948965135213
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.2433049522842103
+      - filter: mlp
+        value: 0.11537153133586801
+      - value: 0.11236945502439658
+  - layer_range: [2, 4]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.13087986863180992
+      - filter: mlp
+        value: 0.05060452788200992
+      - value: 0.029882383396623725
+  - layer_range: [2, 4]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.9938109261305853
+      - filter: mlp
+        value: 0.709432587913349
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.15343343058938377
+      - filter: mlp
+        value: 0.4105917936868785
+      - value: 0.6078632204623161
+  - layer_range: [2, 4]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 0.9634269234020544
+      weight:
+      - filter: self_attn
+        value: 0.03750763360681478
+      - filter: mlp
+        value: 0.29089122858987404
+      - value: 0.3408085857388722
+  - layer_range: [2, 4]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [4, 6]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.8057109303418598
+      - filter: mlp
+        value: 0.9954520808628292
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.02598285706585618
+      - filter: mlp
+        value: 0.06661629726622949
+      - value: 0.1285191000066376
+  - layer_range: [4, 6]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.9112825916608848
+      - filter: mlp
+        value: 0.9322557507910056
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.18823564379986454
+      - filter: mlp
+        value: 0.4552822441636322
+      - value: 0.5120525709221785
+  - layer_range: [4, 6]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.9869122169774399
+      - filter: mlp
+        value: 1.0
+      - value: 0.9751291459565757
+      weight:
+      - filter: self_attn
+        value: 0.00493134813843582
+      - filter: mlp
+        value: 0.3008979965262413
+      - value: 0.2528466849993097
+  - layer_range: [4, 6]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.8956512783019246
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.4197408619693966
+      - filter: mlp
+        value: 0.1448902874618845
+      - value: 0.5196932662212128
+  - layer_range: [4, 6]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [6, 8]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.05321377226808306
+      - filter: mlp
+        value: 0.0482589904702303
+      - value: 0.433407006546336
+  - layer_range: [6, 8]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.8300482882633113
+      - filter: mlp
+        value: 0.8951636861593875
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.35952608658046414
+      - filter: mlp
+        value: 0.17385333183950857
+      - value: 0.6366514725970246
+  - layer_range: [6, 8]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.7848308077099464
+      - filter: mlp
+        value: 0.869549457974157
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.12433943050311849
+      - filter: mlp
+        value: 0.3065832590226165
+      - value: 0.33138948726149514
+  - layer_range: [6, 8]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.11885967308786714
+      - filter: mlp
+        value: 0.29125668567121127
+      - value: 0.19251901269486088
+  - layer_range: [6, 8]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [8, 10]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.9429625513013793
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.4085396076816443
+      - filter: mlp
+        value: 0.038473657720644636
+      - value: 0.35014489493395495
+  - layer_range: [8, 10]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.26957216810533163
+      - filter: mlp
+        value: 0.2393300696241166
+      - value: 0.4735322427351712
+  - layer_range: [8, 10]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.8594757954447017
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.26101395702355007
+      - filter: mlp
+        value: 0.3147672140145126
+      - value: 0.11658182776184756
+  - layer_range: [8, 10]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.6948062341711919
+      - value: 0.9312401427737346
+      weight:
+      - filter: self_attn
+        value: 0.1987774487170517
+      - filter: mlp
+        value: 0.5628384475763534
+      - value: 0.2765378221890683
+  - layer_range: [8, 10]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [10, 12]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.8230035654228713
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.1741591536775035
+      - filter: mlp
+        value: 0.30563583223301516
+      - value: 0.2060419023239155
+  - layer_range: [10, 12]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.9991063013557119
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.1470996125766866
+      - filter: mlp
+        value: 0.06646481892400827
+      - value: 0.2645489609472036
+  - layer_range: [10, 12]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.6812899560643833
+      - filter: mlp
+        value: 0.9083104648631823
+      - value: 0.9730062683598184
+      weight:
+      - filter: self_attn
+        value: 0.14278507832578724
+      - filter: mlp
+        value: 0.3475945971407978
+      - value: 0.40266546962595284
+  - layer_range: [10, 12]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.7047231879232164
+      - filter: mlp
+        value: 0.9148432633716144
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.15341559366405985
+      - filter: mlp
+        value: 0.20047704006010095
+      - value: 0.17364445581398172
+  - layer_range: [10, 12]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [12, 14]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.6974090973508299
+      - filter: mlp
+        value: 1.0
+      - value: 0.9553573565285324
+      weight:
+      - filter: self_attn
+        value: 0.03614401712451334
+      - filter: mlp
+        value: 0.1287785039219736
+      - value: 0.3780545754310749
+  - layer_range: [12, 14]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.7857328784783159
+      - filter: mlp
+        value: 1.0
+      - value: 0.6631303877423032
+      weight:
+      - filter: self_attn
+        value: 0.21728574423632604
+      - filter: mlp
+        value: 0.22813107248290188
+      - value: 0.1435266378249425
+  - layer_range: [12, 14]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.7579910864422339
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.21526786827735228
+      - filter: mlp
+        value: 0.19769619474642783
+      - value: 0.49420458585638627
+  - layer_range: [12, 14]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.8379590665264793
+      - filter: mlp
+        value: 1.0
+      - value: 0.6778673543559375
+      weight:
+      - filter: self_attn
+        value: 0.060679858649663874
+      - filter: mlp
+        value: 0.17248738428562518
+      - value: 0.05145640258269078
+  - layer_range: [12, 14]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [14, 16]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.8193296716327286
+      - value: 0.709644132681917
+      weight:
+      - filter: self_attn
+        value: 0.09821428505487592
+      - filter: mlp
+        value: 0.0039875777021436964
+      - value: 0.27550746634944184
+  - layer_range: [14, 16]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.9420135087156387
+      - filter: mlp
+        value: 1.0
+      - value: 0.9478569230341948
+      weight:
+      - filter: self_attn
+        value: 0.32640822225239857
+      - filter: mlp
+        value: 0.28189746971019747
+      - value: 0.09777040841174603
+  - layer_range: [14, 16]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.9811539353914964
+      - filter: mlp
+        value: 1.0
+      - value: 0.9947034500579488
+      weight:
+      - filter: self_attn
+        value: 0.015308461456516246
+      - filter: mlp
+        value: 0.0018966958379955934
+      - value: 0.24275389952300747
+  - layer_range: [14, 16]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.9022355771447704
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.03331841447575224
+      - filter: mlp
+        value: 0.03561712850019841
+      - value: 0.16096143804589919
+  - layer_range: [14, 16]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [16, 18]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.8813466618200871
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.20435001101909528
+      - filter: mlp
+        value: 0.1516594727144469
+      - value: 0.2269819409999868
+  - layer_range: [16, 18]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 0.8113796412034742
+      weight:
+      - filter: self_attn
+        value: 0.23760349395229585
+      - filter: mlp
+        value: 0.1725436279774783
+      - value: 0.5818814139457673
+  - layer_range: [16, 18]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.9307369835995082
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.0673898519051937
+      - filter: mlp
+        value: 0.049368399457210624
+      - value: 0.2621269048339309
+  - layer_range: [16, 18]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.8219541044757637
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.21320061393511042
+      - filter: mlp
+        value: 0.09188781867337345
+      - value: 0.27266490524762327
+  - layer_range: [16, 18]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [18, 20]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.7993530327131696
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.20420262433348008
+      - filter: mlp
+        value: 0.43400570066910155
+      - value: 0.13720822682656159
+  - layer_range: [18, 20]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 0.7035563346885239
+      weight:
+      - filter: self_attn
+        value: 0.3313523263002212
+      - filter: mlp
+        value: 0.356035051194268
+      - value: 0.4742357680522683
+  - layer_range: [18, 20]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.2475654838180605
+      - filter: mlp
+        value: 0.35095371882044646
+      - value: 0.18536862919946695
+  - layer_range: [18, 20]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.02997204931537696
+      - filter: mlp
+        value: 0.4103581291392323
+      - value: 0.19313933251158066
+  - layer_range: [18, 20]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [20, 22]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 0.5321196166337413
+      weight:
+      - filter: self_attn
+        value: 0.17930537920958298
+      - filter: mlp
+        value: 0.07662274511683252
+      - value: 0.1354315278471591
+  - layer_range: [20, 22]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.3768803907042144
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.1592147705254305
+      - filter: mlp
+        value: 0.18410207999201075
+      - value: 0.4928015910047033
+  - layer_range: [20, 22]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.37897278298418885
+      - filter: mlp
+        value: 0.0952591073533606
+      - value: 0.03551732810121447
+  - layer_range: [20, 22]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.2682334102128691
+      - filter: mlp
+        value: 0.33485781481395227
+      - value: 0.3395139468281392
+  - layer_range: [20, 22]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [22, 24]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.8002588203446623
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.2549204541625693
+      - filter: mlp
+        value: 0.3722418477156178
+      - value: 0.2410463731352089
+  - layer_range: [22, 24]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.9220873255898425
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.487455295718532
+      - filter: mlp
+        value: 0.40022413917173594
+      - value: 0.17846009757502157
+  - layer_range: [22, 24]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.7696341317318985
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.011267799816515114
+      - filter: mlp
+        value: 0.5320959832591042
+      - value: 0.17095406531325266
+  - layer_range: [22, 24]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.556101646343872
+      - filter: mlp
+        value: 0.5470253909079791
+      - value: 0.13241555469863223
+  - layer_range: [22, 24]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [24, 26]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.8667033674916582
+      - filter: mlp
+        value: 1.0
+      - value: 0.9446091486920749
+      weight:
+      - filter: self_attn
+        value: 0.4134110775513897
+      - filter: mlp
+        value: 0.0181822765943834
+      - value: 0.22797659617038232
+  - layer_range: [24, 26]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.9839865829690491
+      - value: 0.8252981103449059
+      weight:
+      - filter: self_attn
+        value: 0.3310295320944009
+      - filter: mlp
+        value: 0.05341478458353629
+      - value: 0.3588847186159219
+  - layer_range: [24, 26]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.8834823812212265
+      - value: 0.8195593509048733
+      weight:
+      - filter: self_attn
+        value: 0.3778012590489552
+      - filter: mlp
+        value: 0.2553204906819882
+      - value: 0.23250565137970108
+  - layer_range: [24, 26]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.7731602497153744
+      - value: 0.8647152680789973
+      weight:
+      - filter: self_attn
+        value: 0.1101209698118704
+      - filter: mlp
+        value: 0.2399169741437055
+      - value: 0.32311925187355206
+  - layer_range: [24, 26]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [26, 28]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.9508674341172941
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.4312865186270921
+      - filter: mlp
+        value: 0.28336325917543326
+      - value: 0.051826325177477234
+  - layer_range: [26, 28]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.8945725432745376
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.03524133636598346
+      - filter: mlp
+        value: 0.21426126710725438
+      - value: 0.31724116335002545
+  - layer_range: [26, 28]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.7138130384877139
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.04890129864608137
+      - filter: mlp
+        value: 0.3324333287494201
+      - value: 0.11533647335498036
+  - layer_range: [26, 28]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 0.9200281327001997
+      weight:
+      - filter: self_attn
+        value: 0.300842776105564
+      - filter: mlp
+        value: 0.08363140003203932
+      - value: 0.2538677006866867
+  - layer_range: [26, 28]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [28, 30]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.7116000185808022
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.10977758983122704
+      - filter: mlp
+        value: 0.1839207861311269
+      - value: 0.5426174846632369
+  - layer_range: [28, 30]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.8412049419861911
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.3517232690814979
+      - filter: mlp
+        value: 0.11878679655495025
+      - value: 0.432611353923264
+  - layer_range: [28, 30]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.7196182744068202
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.29848623969081
+      - filter: mlp
+        value: 0.034661358236493495
+      - value: 0.3438376072572394
+  - layer_range: [28, 30]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.051511204430449285
+      - filter: mlp
+        value: 0.3617968383178797
+      - value: 0.2578690795635758
+  - layer_range: [28, 30]
+    model: ./Yosegi-0601
+- sources:
+  - layer_range: [30, 32]
+    model: mistralai/Mistral-7B-v0.3
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.7971002248466003
+      - filter: mlp
+        value: 0.8931695149333363
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.07401430804790136
+      - filter: mlp
+        value: 0.00696466997386886
+      - value: 0.08295038526296711
+  - layer_range: [30, 32]
+    model: meta-math/MetaMath-Mistral-7B
+    parameters:
+      density:
+      - filter: self_attn
+        value: 1.0
+      - filter: mlp
+        value: 0.8158777337631619
+      - value: 0.8348784699583887
+      weight:
+      - filter: self_attn
+        value: 0.26799125918248423
+      - filter: mlp
+        value: 0.08176923813129498
+      - value: 0.030317330226146508
+  - layer_range: [30, 32]
+    model: uukuguy/speechless-zephyr-code-functionary-7b
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.8188850632365792
+      - filter: mlp
+        value: 0.7463831519693573
+      - value: 0.6515317051533988
+      weight:
+      - filter: self_attn
+        value: 0.21122007850953434
+      - filter: mlp
+        value: 0.1463362342258229
+      - value: 0.09176704194956312
+  - layer_range: [30, 32]
+    model: ./Ninja-v1-RP-expressive-v2-LoRA
+    parameters:
+      density:
+      - filter: self_attn
+        value: 0.9313941807354906
+      - filter: mlp
+        value: 1.0
+      - value: 1.0
+      weight:
+      - filter: self_attn
+        value: 0.1443680121177074
+      - filter: mlp
+        value: 0.08309606396368145
+      - value: 0.37059044424517035
+  - layer_range: [30, 32]
+    model: ./Yosegi-0601

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c76f3665a788a49dac074ef36082a7730d76d45eb491a2d44560c4b7af61c6f
+size 4886547008

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff84a0a9e62b5cd192df4f11dde4ab5dd63e4ca39ab1398c7626ed9234722665
+size 4915916176

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73358c2ea83bf22d73d195d8385436364195060dd0dcde99c92742832ddc57cd
+size 4681034848

model.safetensors.index.json

@@ -0,0 +1 @@
+{"metadata": {"mergekit_version": "0.0.4.2", "total_size": 14483464192}, "weight_map": {"lm_head.weight": "model-00001-of-00003.safetensors", "model.embed_tokens.weight": "model-00001-of-00003.safetensors", "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors", "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors", "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.11.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.11.mlp.down_proj.weight": "model-00001-of-00003.safetensors", "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", "model.layers.11.mlp.up_proj.weight": "model-00001-of-00003.safetensors", "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.12.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.12.mlp.down_proj.weight": "model-00001-of-00003.safetensors", "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", "model.layers.12.mlp.up_proj.weight": "model-00001-of-00003.safetensors", "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.13.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.13.mlp.down_proj.weight": "model-00001-of-00003.safetensors", "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", "model.layers.13.mlp.up_proj.weight": "model-00001-of-00003.safetensors", "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.14.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.14.mlp.down_proj.weight": "model-00001-of-00003.safetensors", "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", "model.layers.14.mlp.up_proj.weight": "model-00001-of-00003.safetensors", "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.15.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.15.mlp.down_proj.weight": "model-00001-of-00003.safetensors", "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", "model.layers.15.mlp.up_proj.weight": "model-00001-of-00003.safetensors", "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.16.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.16.mlp.down_proj.weight": "model-00001-of-00003.safetensors", "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", "model.layers.16.mlp.up_proj.weight": "model-00001-of-00003.safetensors", "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.17.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.17.mlp.down_proj.weight": "model-00001-of-00003.safetensors", "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", "model.layers.17.mlp.up_proj.weight": "model-00001-of-00003.safetensors", "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.18.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.2.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.2.mlp.down_proj.weight": "model-00002-of-00003.safetensors", "model.layers.2.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", "model.layers.2.mlp.up_proj.weight": "model-00002-of-00003.safetensors", "model.layers.2.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.2.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.2.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.2.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.2.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors", "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors", "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors", "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors", "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors", "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors", "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors", "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors", "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.24.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.24.mlp.down_proj.weight": "model-00002-of-00003.safetensors", "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", "model.layers.24.mlp.up_proj.weight": "model-00002-of-00003.safetensors", "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.25.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.25.mlp.down_proj.weight": "model-00002-of-00003.safetensors", "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", "model.layers.25.mlp.up_proj.weight": "model-00002-of-00003.safetensors", "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.26.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.26.mlp.down_proj.weight": "model-00002-of-00003.safetensors", "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", "model.layers.26.mlp.up_proj.weight": "model-00002-of-00003.safetensors", "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.27.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.27.mlp.down_proj.weight": "model-00002-of-00003.safetensors", "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", "model.layers.27.mlp.up_proj.weight": "model-00002-of-00003.safetensors", "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.28.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.28.mlp.down_proj.weight": "model-00002-of-00003.safetensors", "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors", "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors", "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors", "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.3.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.3.mlp.down_proj.weight": "model-00003-of-00003.safetensors", "model.layers.3.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", "model.layers.3.mlp.up_proj.weight": "model-00003-of-00003.safetensors", "model.layers.3.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.3.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.3.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.3.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.3.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors", "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors", "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors", "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors", "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.4.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.4.mlp.down_proj.weight": "model-00003-of-00003.safetensors", "model.layers.4.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", "model.layers.4.mlp.up_proj.weight": "model-00003-of-00003.safetensors", "model.layers.4.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.4.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.4.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.4.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.4.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.5.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.5.mlp.down_proj.weight": "model-00003-of-00003.safetensors", "model.layers.5.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", "model.layers.5.mlp.up_proj.weight": "model-00003-of-00003.safetensors", "model.layers.5.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.5.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.5.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.5.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.5.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.6.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.6.mlp.down_proj.weight": "model-00003-of-00003.safetensors", "model.layers.6.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", "model.layers.6.mlp.up_proj.weight": "model-00003-of-00003.safetensors", "model.layers.6.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.6.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.6.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.6.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.6.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.7.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.7.mlp.down_proj.weight": "model-00003-of-00003.safetensors", "model.layers.7.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", "model.layers.7.mlp.up_proj.weight": "model-00003-of-00003.safetensors", "model.layers.7.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.7.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.7.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.7.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.7.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.8.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.8.mlp.down_proj.weight": "model-00003-of-00003.safetensors", "model.layers.8.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", "model.layers.8.mlp.up_proj.weight": "model-00003-of-00003.safetensors", "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.8.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.8.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.8.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.8.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.9.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.9.mlp.down_proj.weight": "model-00003-of-00003.safetensors", "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", "model.layers.9.mlp.up_proj.weight": "model-00003-of-00003.safetensors", "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.norm.weight": "model-00003-of-00003.safetensors"}}

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true
+}