Hugging Face's logo

deltanym
/
llama-3.1-120b-instruct

Text Generation
Transformers
Safetensors
llama
mergekit
Merge
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
deltanym commited on
Commit
041b299
•
1 Parent(s): 952a501

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +1 -0
  2. README.md +55 -0
  3. config.json +40 -0
  4. mergekit_config.yml +24 -0
  5. model-00001-of-00050.safetensors +3 -0
  6. model-00002-of-00050.safetensors +3 -0
  7. model-00003-of-00050.safetensors +3 -0
  8. model-00004-of-00050.safetensors +3 -0
  9. model-00005-of-00050.safetensors +3 -0
  10. model-00006-of-00050.safetensors +3 -0
  11. model-00007-of-00050.safetensors +3 -0
  12. model-00008-of-00050.safetensors +3 -0
  13. model-00009-of-00050.safetensors +3 -0
  14. model-00010-of-00050.safetensors +3 -0
  15. model-00011-of-00050.safetensors +3 -0
  16. model-00012-of-00050.safetensors +3 -0
  17. model-00013-of-00050.safetensors +3 -0
  18. model-00014-of-00050.safetensors +3 -0
  19. model-00015-of-00050.safetensors +3 -0
  20. model-00016-of-00050.safetensors +3 -0
  21. model-00017-of-00050.safetensors +3 -0
  22. model-00018-of-00050.safetensors +3 -0
  23. model-00019-of-00050.safetensors +3 -0
  24. model-00020-of-00050.safetensors +3 -0
  25. model-00021-of-00050.safetensors +3 -0
  26. model-00022-of-00050.safetensors +3 -0
  27. model-00023-of-00050.safetensors +3 -0
  28. model-00024-of-00050.safetensors +3 -0
  29. model-00025-of-00050.safetensors +3 -0
  30. model-00026-of-00050.safetensors +3 -0
  31. model-00027-of-00050.safetensors +3 -0
  32. model-00028-of-00050.safetensors +3 -0
  33. model-00029-of-00050.safetensors +3 -0
  34. model-00030-of-00050.safetensors +3 -0
  35. model-00031-of-00050.safetensors +3 -0
  36. model-00032-of-00050.safetensors +3 -0
  37. model-00033-of-00050.safetensors +3 -0
  38. model-00034-of-00050.safetensors +3 -0
  39. model-00035-of-00050.safetensors +3 -0
  40. model-00036-of-00050.safetensors +3 -0
  41. model-00037-of-00050.safetensors +3 -0
  42. model-00038-of-00050.safetensors +3 -0
  43. model-00039-of-00050.safetensors +3 -0
  44. model-00040-of-00050.safetensors +3 -0
  45. model-00041-of-00050.safetensors +3 -0
  46. model-00042-of-00050.safetensors +3 -0
  47. model-00043-of-00050.safetensors +3 -0
  48. model-00044-of-00050.safetensors +3 -0
  49. model-00045-of-00050.safetensors +3 -0
  50. model-00046-of-00050.safetensors +3 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model:
3
+ - meta-llama/Llama-3.1-70B-Instruct
4
+ library_name: transformers
5
+ tags:
6
+ - mergekit
7
+ - merge
8
+
9
+ ---
10
+ # llama-3.1-120b-instruct
11
+
12
+ This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
13
+ It's a recreation of [mlabonne/Meta-Llama-3-120B-Instruct](https://huggingface.co/mlabonne/Meta-Llama-3-120B-Instruct) but using Llama 3.1 70b instead of Llama 3, with the same configuration.
14
+
15
+ ## Merge Details
16
+ ### Merge Method
17
+
18
+ This model was merged using the passthrough merge method.
19
+
20
+ ### Models Merged
21
+
22
+ The following models were included in the merge:
23
+ * [meta-llama/Llama-3.1-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct)
24
+
25
+ ### Configuration
26
+
27
+ The following YAML configuration was used to produce this model:
28
+
29
+ ```yaml
30
+ slices:
31
+ - sources:
32
+ - layer_range: [0, 20]
33
+ model: meta-llama/Llama-3.1-70B-Instruct
34
+ - sources:
35
+ - layer_range: [10, 30]
36
+ model: meta-llama/Llama-3.1-70B-Instruct
37
+ - sources:
38
+ - layer_range: [20, 40]
39
+ model: meta-llama/Llama-3.1-70B-Instruct
40
+ - sources:
41
+ - layer_range: [30, 50]
42
+ model: meta-llama/Llama-3.1-70B-Instruct
43
+ - sources:
44
+ - layer_range: [40, 60]
45
+ model: meta-llama/Llama-3.1-70B-Instruct
46
+ - sources:
47
+ - layer_range: [50, 70]
48
+ model: meta-llama/Llama-3.1-70B-Instruct
49
+ - sources:
50
+ - layer_range: [60, 80]
51
+ model: meta-llama/Llama-3.1-70B-Instruct
52
+ merge_method: passthrough
53
+ dtype: float16
54
+
55
+ ```
config.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "meta-llama/Llama-3.1-70B-Instruct",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 128000,
9
+ "eos_token_id": [
10
+ 128001,
11
+ 128008,
12
+ 128009
13
+ ],
14
+ "head_dim": 128,
15
+ "hidden_act": "silu",
16
+ "hidden_size": 8192,
17
+ "initializer_range": 0.02,
18
+ "intermediate_size": 28672,
19
+ "max_position_embeddings": 131072,
20
+ "mlp_bias": false,
21
+ "model_type": "llama",
22
+ "num_attention_heads": 64,
23
+ "num_hidden_layers": 140,
24
+ "num_key_value_heads": 8,
25
+ "pretraining_tp": 1,
26
+ "rms_norm_eps": 1e-05,
27
+ "rope_scaling": {
28
+ "factor": 8.0,
29
+ "high_freq_factor": 4.0,
30
+ "low_freq_factor": 1.0,
31
+ "original_max_position_embeddings": 8192,
32
+ "rope_type": "llama3"
33
+ },
34
+ "rope_theta": 500000.0,
35
+ "tie_word_embeddings": false,
36
+ "torch_dtype": "float16",
37
+ "transformers_version": "4.46.0",
38
+ "use_cache": true,
39
+ "vocab_size": 128256
40
+ }
mergekit_config.yml ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ slices:
2
+ - sources:
3
+ - layer_range: [0, 20]
4
+ model: meta-llama/Llama-3.1-70B-Instruct
5
+ - sources:
6
+ - layer_range: [10, 30]
7
+ model: meta-llama/Llama-3.1-70B-Instruct
8
+ - sources:
9
+ - layer_range: [20, 40]
10
+ model: meta-llama/Llama-3.1-70B-Instruct
11
+ - sources:
12
+ - layer_range: [30, 50]
13
+ model: meta-llama/Llama-3.1-70B-Instruct
14
+ - sources:
15
+ - layer_range: [40, 60]
16
+ model: meta-llama/Llama-3.1-70B-Instruct
17
+ - sources:
18
+ - layer_range: [50, 70]
19
+ model: meta-llama/Llama-3.1-70B-Instruct
20
+ - sources:
21
+ - layer_range: [60, 80]
22
+ model: meta-llama/Llama-3.1-70B-Instruct
23
+ merge_method: passthrough
24
+ dtype: float16
model-00001-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2d6b2b03306e00c0e70d58598d4911b587f7f7537c7e33137aed53858092bf8e
3
+ size 4672471496
model-00002-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:daf2022d1fc8804e5fd01c9dcc15a2ba9807c42d06cf5066de5505a6c5c1ca6d
3
+ size 4831922680
model-00003-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7bb14258180ae65798f4c1d09f55d793f81d8563e8831a1ea6b8ea4799848208
3
+ size 4966190736
model-00004-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2264e94f6b4bffb614e6733922771b07b2537bf24f67ed5ff4fe0e7b11b5203
3
+ size 4831906200
model-00005-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:520e6206ce7e53029587bae3f2fd1bb7c6c3471635677750853186493da1f23f
3
+ size 4999711712
model-00006-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2859ef4d85f9704d9cee7d8ff64e994b164cdaa5b56d0e110d12c9ca9522abad
3
+ size 4798385224
model-00007-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:595043aa527c3c44346348e59ecf0aeb83d070ca8fa2be54e6a3551be0d76130
3
+ size 4831906200
model-00008-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a9f3551f16c072dba382dac464a5d906df452b2350e098adbf0f007265448e40
3
+ size 4966190736
model-00009-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4a49fba98efd43aa2b1ad873f12a9d9046e61b8c9606b26e54daacefb596fb0d
3
+ size 4664167384
model-00010-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:34884dbc8c6dcf962212f69c74f872c5797f89833fddb70b120b497ce4c3bcc3
3
+ size 4831906200
model-00011-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:35252e9be0ce4f80e3c2d6ff8d2701be1572903d99269d87112f08aa6f3a4ff7
3
+ size 4999711712
model-00012-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9d30389aaf95e80ae5b1c93bcf62d499a18ce68da09ccaf1ca0a67582bff7261
3
+ size 4798385224
model-00013-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fd2e6aaa1024646476eb7c97ca032bcea82b0559b1ea7683e88b862decaa9fd3
3
+ size 4831906200
model-00014-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2d4e77a1583a7bdfc25680bc08372b58cb87c885f0264036f6c4082b5dd54af2
3
+ size 4966190736
model-00015-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:76416e5634cadbaa71222241610194bc69f38c70334183b0456181421ac1efd9
3
+ size 4831906200
model-00016-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff44e489e7cc25b0f45fc91ec702df1108ed9844bbf1fe8090965b20445ba60a
3
+ size 4966174232
model-00017-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5568d1a7d9287079145ed0878780056904c1ba64c0fa7cd3a5aae6f502272681
3
+ size 4966140544
model-00018-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ebdeccc43575e6a538ddae18085ea6879ff429654d07290e377de5276a7fe471
3
+ size 4999711704
model-00019-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:345b97b98f0fad656d8062288dc1e6db353c60f478114d07ae394908ff84d432
3
+ size 4999711712
model-00020-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a16747f24d34d3e19b263f41a92e94e6bde4f0a7ec778e95dbfc4fa041b6ebae
3
+ size 4798385224
model-00021-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d4fdbdad88200c43381bfe67f589c3c70e0d59c4e0eb2b9c7ede06238f82c363
3
+ size 4831906200
model-00022-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:32c8e0dfa125ecfffb762fdcda09a64097125c6987887b79517f7ba806c37a82
3
+ size 4966190736
model-00023-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e4da4c885e4a3b4dcd0c659619be30ae37a65856aa566b6be9117ae66e4fcf32
3
+ size 4831906200
model-00024-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9f1b0dfd73ab6865f400ec295fde62f00ff6c3ef045d922d2c34391fedb5d2b2
3
+ size 4664167376
model-00025-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:57e1c679cd658852974773b35a7f7ae14beda6096528d29e62d8a40b0f9264a1
3
+ size 4999711712
model-00026-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3240e649c292d905f0698ca896f1dfac4844a9e48c0d2474a3dfe9d0d2412bbe
3
+ size 4798385224
model-00027-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff438d7da66ed3049d08ed6c501e011dbfbbf1f57085903f42ed4d875dbbda3e
3
+ size 4831906200
model-00028-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8ac89613cd93016b780dbff0290bbbc614e9f7610e42f238e37420661631bae2
3
+ size 4966190736
model-00029-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0dbd77d2ee348362ac012aba2eef2a0fd54b256ffb40ccf9e6d6d9146cab0d2
3
+ size 4831906200
model-00030-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:715f6e63ffca34def2a8e271c7fa6d30345fd99147c86991f7d9249b95849803
3
+ size 4966190736
model-00031-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:168bab91302186fc04b8e87ae389f5ee6e63cb220cfb273b1b2873c90fcab20b
3
+ size 4982901368
model-00032-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d3db5ed93d9902bef4b3c5014433ff8258f3aa2f5a5619b42dedb8814f4474a8
3
+ size 4982934368
model-00033-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f548efaa6c8fc1ad7e54e010e88d166f2a0555062cfe8bd7ea4d972d277c3301
3
+ size 4999711720
model-00034-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:62475c0bac2a8fd6f7ec6bdf8eb9655857c88f06183384beca3297031c5b5366
3
+ size 4798385240
model-00035-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c4b73824be4668f7487d5eb249445dd30450e05e083dced0687b8734078bfb91
3
+ size 4831906208
model-00036-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6fef95f1be960bf78af29cf1aeb5dad16fdb62813ebf2b75ced025c0d46ea8f1
3
+ size 4966190744
model-00037-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:72a9a30521d09c8f5e417621b08e89d266075382743c047d418951b9594eeb82
3
+ size 4831906208
model-00038-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:56b3568aafcb4a3d302619ca3f2f8ad6b32dada5c637a666c1aa577b04e58fe1
3
+ size 4966174256
model-00039-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a81a33c36ee2a2553537fea5714b020061a5b1e5f0201a6c4e622adef8d484a8
3
+ size 4966140544
model-00040-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b86c3ebdee6c199c413afed34e453399bec1befde451f9568933e166d56e3a54
3
+ size 4999711728
model-00041-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe048112581f18ab2b711f986330edcc00f4f90ce5ca870e187aab437b1a689c
3
+ size 4999711744
model-00042-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a907a87e173ab30cb6d7856626e210f23f01eac564bd4696d274abf9dac60f8e
3
+ size 4798385256
model-00043-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4ec7ccb8b8564bab41b16cb34808c73d331c5759a640d0102cd5416fe25d66f8
3
+ size 4831906216
model-00044-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2eadae2b8bf98f2c17505931c54bbaab78530eadf17ffff588ab125992b2b2b3
3
+ size 4966190768
model-00045-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d132c7cb3612a6ade0c2e3406d1bbb18d277df83a00ee4023249899ac86db6c
3
+ size 4831906224
model-00046-of-00050.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:685b0145bd48929b1dacdae3e7c807541ec003e81bd10c55152806939d1b28ea
3
+ size 4966174256