metadata
license: apache-2.0
https://wandb.ai/open-assistant/supervised-finetuning/runs/7pz5n33h exported checkpoint: 4500 steps
datasets:
oasst_export_eu:
datasets:
- oasst_export:
lang: "en,es,de,fr"
input_file_path: 2023-03-27_oasst_research_ready_synth.jsonl.gz
- alpaca
- oig_file:
source_url: https://huggingface.co/datasets/laion/OIG/resolve/main/unified_chip2.jsonl
max_count: 15000
min_length: 500
val_split: 0.2
- oig_file:
source_url: https://huggingface.co/datasets/laion/OIG/raw/main/unified_grade_school_math_instructions.jsonl
val_split: 0.1
min_length: 1000
sort_by_length: false
use_custom_sampler: false
pythia:
pythia-12b:
fp16: true
log_dir: "pythia_log_12b"
learning_rate: 6e-6
model_name: EleutherAI/pythia-12b-deduped
output_dir: pythia_model_12b
weight_decay: 0.0
residual_dropout: 0.2
max_length: 2048
use_flash_attention: true
warmup_steps: 100
gradient_checkpointing: false
gradient_accumulation_steps: 4
per_device_train_batch_size: 2
per_device_eval_batch_size: 5
eval_steps: 200
save_steps: 500
num_train_epochs: 16
save_total_limit: 4