laneformer-2b-it / config.json
mg-kog's picture
Publich Laneformer 1B instruct
b4f40ad verified
Raw History Blame Contribute Delete
1.47 kB
{
"model_type": "laneformer",
"architectures": [
"LaneformerForCausalLM"
],
"auto_map": {
"AutoConfig": "configuration_laneformer.LaneformerConfig",
"AutoModel": "modeling_laneformer.LaneformerModel",
"AutoModelForCausalLM": "modeling_laneformer.LaneformerForCausalLM"
},
"hidden_size": 3072,
"num_hidden_layers": 15,
"num_attention_heads": 32,
"num_key_value_heads": 16,
"intermediate_size": 12288,
"max_position_embeddings": 4096,
"max_seq_len": 4096,
"eos_token_id": 2,
"pad_token_id": 2,
"vocab_size": 32000,
"rope_theta": 10000,
"norm_eps": 1e-05,
"use_cache": false,
"num_lanes": 8,
"broadcast_delay": 2,
"use_attention_comm": true,
"use_mlp_comm": true,
"use_early_comm": false,
"lm_head_type": "vocab_parallel",
"pre_norm_lane_agg": false,
"replicated_rmsn_scale": true,
"tie_word_embeddings": false,
"sliding_window": 2048,
"swa_layers": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
],
"layer_types": [
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention"
],
"_attn_implementation": "sdpa",
"dtype": "bfloat16",
"transformers_version": "4.57.6"
}