Upload folder using huggingface_hub

README.md

@@ -0,0 +1,117 @@
+---
+language:
+- en
+license: apache-2.0
+datasets:
+- teknium/OpenHermes-2.5
+base_model: mistral-community/Mixtral-8x22B-v0.1
+tags:
+- exl2
+---
+
+# mixtral-8x22b-instruct-oh - EXL2 2.75bpw
+
+This is a 2.75bpw EXL2 quant of [fireworks-ai/mixtral-8x22b-instruct-oh](https://huggingface.co/fireworks-ai/mixtral-8x22b-instruct-oh)
+
+Details about the model can be found at the above model page.
+
+## EXL2 Version
+
+These quants were made with exllamav2 version 0.0.18. Quants made on this version of EXL2 may not work on older versions of the exllamav2 library.
+
+If you have problems loading these models, please update Text Generation WebUI to the latest version.
+
+## Perplexity Scoring
+
+Below are the perplexity scores for the EXL2 models. A lower score is better. 
+
+_TODO_
+
+### Perplexity Script
+
+This was the script used for perplexity testing.
+
+```bash
+#!/bin/bash
+
+# Activate the conda environment
+source ~/miniconda3/etc/profile.d/conda.sh
+conda activate exllamav2
+
+# Set the model name and bit size
+MODEL_NAME="mixtral-8x22b-instruct-oh"
+BIT_PRECISIONS=(7.0 6.0 5.5 5.0 4.5 4.0 3.5 3.0 2.75 2.5 2.25)
+
+# Print the markdown table header
+echo "| Quant Level | Perplexity Score |"
+echo "|-------------|------------------|"
+
+for BIT_PRECISION in "${BIT_PRECISIONS[@]}"
+do
+  MODEL_DIR="models/${MODEL_NAME}_exl2_${BIT_PRECISION}bpw"
+  if [ -d "$MODEL_DIR" ]; then
+    output=$(python test_inference.py -m "$MODEL_DIR" -gs 44,48 -ed data/wikitext/wikitext-2-v1.parquet)
+    score=$(echo "$output" | grep -oP 'Evaluation perplexity: \K[\d.]+')
+    echo "| $BIT_PRECISION | $score |"
+  fi
+done
+```
+
+
+## Quant Details
+
+This is the script used for quantization.
+
+```bash
+#!/bin/bash
+
+# Activate the conda environment
+source ~/miniconda3/etc/profile.d/conda.sh
+conda activate exllamav2
+
+# Set the model name and bit size
+MODEL_NAME="mixtral-8x22b-instruct-oh"
+
+# Define variables
+MODEL_DIR="models/$MODEL_NAME"
+OUTPUT_DIR="exl2_$MODEL_NAME"
+MEASUREMENT_FILE="measurements/$MODEL_NAME.json"
+
+# Create the measurement file if needed
+if [ ! -f "$MEASUREMENT_FILE" ]; then
+    echo "Creating $MEASUREMENT_FILE"
+    # Create directories
+    if [ -d "$OUTPUT_DIR" ]; then
+        rm -r "$OUTPUT_DIR"
+    fi
+    mkdir "$OUTPUT_DIR"
+
+    python convert.py -i $MODEL_DIR -o $OUTPUT_DIR -nr -om $MEASUREMENT_FILE
+fi
+
+# Choose one of the below. Either create a single quant for testing or a batch of them.
+# BIT_PRECISIONS=(2.25)
+BIT_PRECISIONS=(7.0 6.0 5.5 5.0 4.5 4.0 3.5 3.0 2.75 2.5 2.25)
+
+for BIT_PRECISION in "${BIT_PRECISIONS[@]}"
+do
+    CONVERTED_FOLDER="models/${MODEL_NAME}_exl2_${BIT_PRECISION}bpw"
+
+    # If it doesn't already exist, make the quant
+    if [ ! -d "$CONVERTED_FOLDER" ]; then
+
+        echo "Creating $CONVERTED_FOLDER"
+
+        # Create directories
+        if [ -d "$OUTPUT_DIR" ]; then
+            rm -r "$OUTPUT_DIR"
+        fi
+        mkdir "$OUTPUT_DIR"
+        mkdir "$CONVERTED_FOLDER"
+        
+        # Run conversion commands  
+        python convert.py -i $MODEL_DIR -o $OUTPUT_DIR -nr -m $MEASUREMENT_FILE -b $BIT_PRECISION -cf $CONVERTED_FOLDER
+
+    fi
+done
+```

config.json

@@ -0,0 +1,42 @@
+{
+    "_name_or_path": "mistral-community/Mixtral-8x22B-v0.1",
+    "architectures": [
+        "MixtralForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "hidden_act": "silu",
+    "hidden_size": 6144,
+    "initializer_range": 0.02,
+    "intermediate_size": 16384,
+    "max_position_embeddings": 65536,
+    "model_type": "mixtral",
+    "num_attention_heads": 48,
+    "num_experts_per_tok": 2,
+    "num_hidden_layers": 56,
+    "num_key_value_heads": 8,
+    "num_local_experts": 8,
+    "output_router_logits": false,
+    "rms_norm_eps": 1e-05,
+    "rope_theta": 1000000,
+    "router_aux_loss_coef": 0.001,
+    "router_jitter_noise": 0.0,
+    "sliding_window": null,
+    "tie_word_embeddings": false,
+    "torch_dtype": "float16",
+    "transformers_version": "4.39.3",
+    "use_cache": true,
+    "vocab_size": 32000,
+    "quantization_config": {
+        "quant_method": "exl2",
+        "version": "0.0.18",
+        "bits": 2.75,
+        "head_bits": 6,
+        "calibration": {
+            "rows": 100,
+            "length": 2048,
+            "dataset": "(default)"
+        }
+    }
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.39.3"
+}

output-00001-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c85f2b28f5acd94945484e0ca453270d19dc2cf071006190586fc43bcf66bc4
+size 8590049976

output-00002-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0911d9597cb78319a49d9420e0c82796e2f91e4ea4596da3476949efb610581e
+size 8588598128

output-00003-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d7b101c47ed55d7a208798ab150299e79520d10c7d08a0660f20270b7b1bbf2
+size 8559958840

output-00004-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:292cb9b5fb74fe9dc3a0f0d7fe820a587081419867b8b067b6a6a76791a16011
+size 8590014416

output-00005-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b78392b4c87e0871655c775dd736c170c41facd7b0322800e6ba332a51f2293f
+size 8562505720

output-00006-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:949bc0ad624dac5acb4e99e55f136c3d4d85c9e4ac13bf571694dce8d2d324db
+size 5866246592

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{%- set message_roles = ['USER', 'ASSISTANT'] -%}\n{%- set ns = namespace(initial_system_message_handled=false, messages=messages) -%}\n{{ bos_token }}\n{%- for message in ns.messages -%}\n  {%- if message['role'] | upper == 'SYSTEM' and not ns.initial_system_message_handled -%}\n    {%- set ns.initial_system_message_handled = true -%}\n    {{'SYSTEM: ' + message['content'] }}\n  {%- elif message['role'] | upper != 'SYSTEM' -%}\n    {%- if (message['role'] | upper == 'USER') != ((loop.index0 - (1 if ns.initial_system_message_handled else 0)) % 2 == 0) -%}\n      {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}\n    {%- endif -%}\n    {%- if loop.index0 > 0 or ns.initial_system_message_handled -%}\n      {{ '\\n\\n' }}\n    {%- endif -%}\n    {%- if message['role'] | upper == 'USER' -%}\n      {{ 'USER: ' + message['content'] }}\n    {%- elif message['role'] | upper == 'ASSISTANT' -%}\n      {{ 'ASSISTANT: ' + message['content'] + eos_token}}\n    {%- endif -%}\n  {%- endif -%}\n{%- endfor -%}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<unk>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}