File size: 3,956 Bytes
03e422e 0516871 03e422e 565e35d b221faf 67f7e5b b221faf 0516871 b221faf e732c45 b221faf 0516871 67f7e5b 0516871 67f7e5b 0516871 67f7e5b 0516871 67f7e5b e732c45 67f7e5b fba637b 67f7e5b fba637b 67f7e5b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 |
---
license: apache-2.0
language:
- en
---
## Aura-8B
## Introduction
**Aura-8B** is a state of the art dedicated roleplaying model designed to fulfill your every desire.
This finetune has seen several hundreds of millions of tokens of instruction and roleplaying data.
Developed by **Aura Industries**, with contributions from **Anthracite Org**
## Model Details
- **Model Name**: Aura-8B
- **Base Model**: [arcee-ai/Llama-3.1-SuperNova-Lite](https://huggingface.co/arcee-ai/Llama-3.1-SuperNova-Lite)
- **Model Type**: Chat Completions
- **Prompt Format**: Llama 3
- **License**: Apache-2.0
- **Language**: English
- **Max Context**: 8,192+ tokens
## Training Configuration
```
base_model: arcee-ai/Llama-3.1-SuperNova-Lite
model_type: AutoModelForCausalLM
tokenizer_type: AutoTokenizer
load_in_8bit: false
load_in_4bit: false
strict: false
datasets:
- path: FourOhFour/RP_Phase
type: chat_template
chat_template: llama3
roles_to_train: ["gpt"]
field_messages: conversations
message_field_role: from
message_field_content: value
train_on_eos: turn
- path: Nitral-AI/Cybersecurity-ShareGPT
type: chat_template
chat_template: llama3
roles_to_train: ["gpt"]
field_messages: conversations
message_field_role: from
message_field_content: value
train_on_eos: turn
- path: Nitral-AI/Medical_Instruct-ShareGPT
type: chat_template
chat_template: llama3
roles_to_train: ["gpt"]
field_messages: conversations
message_field_role: from
message_field_content: value
train_on_eos: turn
- path: Nitral-AI/Olympiad_Math-ShareGPT
type: chat_template
chat_template: llama3
roles_to_train: ["gpt"]
field_messages: conversations
message_field_role: from
message_field_content: value
train_on_eos: turn
- path: NewEden/Claude-Instruct-5k
type: chat_template
chat_template: llama3
roles_to_train: ["gpt"]
field_messages: conversations
message_field_role: from
message_field_content: value
train_on_eos: turn
- path: lodrick-the-lafted/kalo-opus-instruct-3k-filtered
type: chat_template
chat_template: llama3
roles_to_train: ["gpt"]
field_messages: conversations
message_field_role: from
message_field_content: value
train_on_eos: turn
- path: Nitral-AI/Creative_Writing-ShareGPT
type: chat_template
chat_template: llama3
roles_to_train: ["gpt"]
field_messages: conversations
message_field_role: from
message_field_content: value
train_on_eos: turn
- path: jeiku/Writing
type: completion
field: text
shuffle_merged_datasets: true
dataset_prepared_path:
val_set_size: 0.01
output_dir: ./output/out
hub_model_id: jeiku/Aura-8B
hub_strategy: "all_checkpoints"
push_dataset_to_hub:
hf_use_auth_token: true
sequence_len: 8192
sample_packing: true
eval_sample_packing: false
pad_to_sequence_len:
wandb_project: Aura-8B
wandb_entity:
wandb_watch:
wandb_name: Aura-8B
wandb_log_model:
gradient_accumulation_steps: 16
micro_batch_size: 2
num_epochs: 2
optimizer: paged_adamw_8bit
lr_scheduler: cosine
learning_rate: 1e-5
train_on_inputs: false
group_by_length: false
bf16: auto
fp16:
tf32: false
gradient_checkpointing: true
early_stopping_patience:
resume_from_checkpoint:
local_rank:
logging_steps: 1
xformers_attention:
flash_attention: true
warmup_ratio: 0.1
evals_per_epoch: 2
eval_table_size:
eval_max_new_tokens:
saves_per_epoch: 1
debug:
deepspeed:
weight_decay: 0.05
fsdp:
fsdp_config:
special_tokens:
pad_token: <|finetune_right_pad_id|>
eos_token: <|eot_id|>
```
## License
This model is licensed under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0).
# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard)
Awaiting results... |