andreaskoepf/oasst-sft-3-pythia-12b-epoch-3.5

https://wandb.ai/open-assistant/supervised-finetuning/runs/7pz5n33h exported checkpoint: 4500 steps

datasets:

oasst_export_eu:
  datasets:
    - oasst_export:
        lang: "en,es,de,fr"
        input_file_path: 2023-03-27_oasst_research_ready_synth.jsonl.gz
    - alpaca
    - oig_file:
        source_url: https://huggingface.co/datasets/laion/OIG/resolve/main/unified_chip2.jsonl
        max_count: 15000
        min_length: 500
        val_split: 0.2
    - oig_file:
        source_url: https://huggingface.co/datasets/laion/OIG/raw/main/unified_grade_school_math_instructions.jsonl
        val_split: 0.1
        min_length: 1000
  sort_by_length: false
  use_custom_sampler: false

pythia:

pythia-12b:
  fp16: true
  log_dir: "pythia_log_12b"
  learning_rate: 6e-6
  model_name: EleutherAI/pythia-12b-deduped
  output_dir: pythia_model_12b
  weight_decay: 0.0
  residual_dropout: 0.2
  max_length: 2048
  use_flash_attention: true
  warmup_steps: 100
  gradient_checkpointing: false
  gradient_accumulation_steps: 4
  per_device_train_batch_size: 2
  per_device_eval_batch_size: 5
  eval_steps: 200
  save_steps: 500
  num_train_epochs: 16
  save_total_limit: 4