config.json

{
 "vocab_size": 152064,
 "max_position_embeddings": 32768,
 "hidden_size": 3584,
 "intermediate_size": 18944,
 "num_hidden_layers": 32,
 "num_attention_heads": 28,
 "use_sliding_window": false,
 "sliding_window": 131072,
 "max_window_layers": 28,
 "num_key_value_heads": 4,
 "hidden_act": "silu",
 "initializer_range": 0.02,
 "rms_norm_eps": 1e-06,
 "use_cache": false,
 "rope_theta": 1000000.0,
 "attention_dropout": 0.0,
 "return_dict": true,
 "output_hidden_states": false,
 "output_attentions": false,
 "torchscript": false,
 "torch_dtype": "bfloat16",
 "use_bfloat16": false,
 "tf_legacy_loss": false,
 "pruned_heads": {},
 "tie_word_embeddings": false,
 "chunk_size_feed_forward": 0,
 "is_encoder_decoder": false,
 "is_decoder": false,
 "cross_attention_hidden_size": null,
 "add_cross_attention": false,
 "tie_encoder_decoder": false,
 "max_length": 20,
 "min_length": 0,
 "do_sample": false,
 "early_stopping": false,
 "num_beams": 1,
 "num_beam_groups": 1,
 "diversity_penalty": 0.0,
 "temperature": 1.0,
 "top_k": 50,
 "top_p": 1.0,
 "typical_p": 1.0,
 "repetition_penalty": 1.0,
 "length_penalty": 1.0,
 "no_repeat_ngram_size": 0,
 "encoder_no_repeat_ngram_size": 0,
 "bad_words_ids": null,
 "num_return_sequences": 1,
 "output_scores": false,
 "return_dict_in_generate": false,
 "forced_bos_token_id": null,
 "forced_eos_token_id": null,
 "remove_invalid_values": false,
 "exponential_decay_length_penalty": null,
 "suppress_tokens": null,
 "begin_suppress_tokens": null,
 "architectures": [
  "Qwen2ForCausalLM"
 ],
 "finetuning_task": null,
 "id2label": {
  "0": "LABEL_0",
  "1": "LABEL_1"
 },
 "label2id": {
  "LABEL_0": 0,
  "LABEL_1": 1
 },
 "tokenizer_class": null,
 "prefix": null,
 "bos_token_id": 151643,
 "pad_token_id": null,
 "eos_token_id": 151645,
 "sep_token_id": null,
 "decoder_start_token_id": null,
 "task_specific_params": null,
 "problem_type": null,
 "_name_or_path": "rubra-ai/Qwen2-7B-Instruct",
 "transformers_version": "4.42.4",
 "model_type": "qwen2",
 "num_experts": 2,
 "num_experts_per_tok": 2,
 "router_layers": [
  "gate_proj",
  "up_proj",
  "down_proj"
 ],
 "router_layers_index": [
  0,
  1,
  2,
  3,
  4,
  5,
  6,
  7,
  8,
  9,
  10,
  11,
  12,
  13,
  14,
  15,
  16,
  17,
  18,
  19,
  20,
  21,
  22,
  23,
  24,
  25,
  26,
  27,
  28,
  29,
  30,
  31
 ]
}