Download config.json from trymirai/Muse-Glimmer-30B-L: direct link, hf CLI and curl.
- Browser
- Download file 240 kB
-
https://huggingface.co/trymirai/Muse-Glimmer-30B-L/resolve/main/config.json
- Command line
-
hf download hf://trymirai/Muse-Glimmer-30B-L/config.json
-
curl -L -o config.json https://huggingface.co/trymirai/Muse-Glimmer-30B-L/resolve/main/config.json
240 kB
{
"type": "LanguageModelConfig",
"token_codec_config": {
"type": "ChatCodecConfig",
"prompt_template": "{#\n Template: Muse Glimmer ATEM Chat Template\n Renders the ATEM tool-calling protocol: reasoning channel (to=self), tool\n channels (to=<tool>), and the user channel, plus tool definitions and the\n valid-recipient list in the system block.\n\n Whitespace note: every tag uses the {%- -%} / {{- -}} stripping markers, so\n the indentation below is purely for readability and contributes nothing to\n the rendered output.\n#}\n{%- macro render_content(content) -%}\n {%- if content is string -%}\n {{- content -}}\n {%- elif content is not none -%}\n {%- for part in content -%}\n {%- if part['type'] == 'image' -%}\n {{- '<|patch|>' -}}\n {%- elif part['type'] == 'video' -%}\n {{- '<|video|>' -}}\n {%- elif part['type'] == 'text' -%}\n {{- part['text'] -}}\n {%- endif -%}\n {%- endfor -%}\n {%- endif -%}\n{%- endmacro -%}\n{%- macro render_atem(tc) -%}\n {%- set args = tc.function.arguments -%}\n {%- if args is not mapping -%}\n {{- raise_exception('Muse Glimmer ATEM chat template requires tool_call.function.arguments to be a dict (mapping); a JSON string cannot be parsed in the HF jinja sandbox.') -}}\n {%- endif -%}\n {{- '<atem:function_calls>\\n<atem:invoke name=\"' + tc.function.name + '\">\\n' -}}\n {%- for k, v in args.items() -%}\n {{- '<atem:parameter name=\"' + k + '\">' -}}\n {%- if v is boolean -%}\n {%- if v -%}\n true\n {%- else -%}\n false\n {%- endif -%}\n {%- elif v is none -%}\n null\n {%- elif v is mapping or (v is iterable and v is not string) -%}\n {{- v | tojson -}}\n {%- else -%}\n {{- v -}}\n {%- endif -%}\n {{- '</atem:parameter>\\n' -}}\n {%- endfor -%}\n {{- '</atem:invoke>\\n</atem:function_calls>' -}}\n{%- endmacro -%}\n{%- macro render_tool_defs(tools) -%}\n {{- 'In this environment you have access to a set of tools you can use to answer the user\\'s question.\\n\\n' -}}\n {{- 'You can invoke a function by writing a \"<atem:function_calls>\" block like the following:\\n' -}}\n {{- '<atem:function_calls>\\n<atem:invoke name=\"$FUNCTION_NAME\">\\n<atem:parameter name=\"$PARAMETER_NAME\">$PARAMETER_VALUE</atem:parameter>\\n...\\n</atem:invoke>\\n</atem:function_calls>\\n\\n' -}}\n {{- 'String and scalar parameters should be specified as is, while lists and objects should use JSON format. Note that spaces for string values are not stripped. The output is not expected to be valid XML and is parsed with regular expressions.\\n' -}}\n {{- 'Here are the functions available in JSONSchema format:\\n' -}}\n {{- '// Tool metadata\\n' -}}\n {%- set nsns = namespace(seen=[]) -%}\n {%- for tool in tools -%}\n {%- set fn = tool.function if tool.function is defined else tool -%}\n {%- set tns = fn.name.split('.')[0] -%}\n {%- if tns not in nsns.seen -%}\n {%- set nsns.seen = nsns.seen + [tns] -%}\n {%- endif -%}\n {%- endfor -%}\n {%- set nd = tool_namespace_descriptions if tool_namespace_descriptions is defined else {} -%}\n {%- for tns in nsns.seen -%}\n {{- '{\"name\": ' + (tns | tojson) + ', \"description\": ' + ((nd[tns] if tns in nd else '') | tojson) + '}\\n' -}}\n {%- endfor -%}\n {{- '// Function schemas' -}}\n {%- for tool in tools -%}\n {%- set fn = tool.function if tool.function is defined else tool -%}\n {{- '\\n{\"name\": ' + (fn.name | tojson) + ', \"description\": ' + (fn.description | tojson) + ', \"parameters\": ' + (fn.parameters | tojson) + '}' -}}\n {%- endfor -%}\n {{- '\\n\\nHere\\'s an example of how to call a function in the tool set:\\n' -}}\n {{- '(If the tool namespace is not specified, invoke the function directly as `example_function_name` rather than `example_tool_name.example_function_name`)\\n\\n' -}}\n {{- 'to=example_tool_name.example_function_name\\n\\n' -}}\n {{- '<atem:function_calls>\\n<atem:invoke name=\"example_tool_name.example_function_name\">\\n' -}}\n {{- '<atem:parameter name=\"example_parameter_1\">value_1</atem:parameter>\\n' -}}\n {{- '<atem:parameter name=\"example_parameter_2\">This is the value for the second parameter\\nthat can span\\n\"multiple\" lines\\n</atem:parameter>\\n' -}}\n {{- '</atem:invoke>\\n</atem:function_calls>' -}}\n{%- endmacro -%}\n{%- macro render_reasoning() -%}\n {%- set rs = reasoning_strength if reasoning_strength is defined and reasoning_strength else 'high' -%}\n {{- 'Reasoning strength: ' + rs + '.' -}}\n{%- endmacro -%}\n{%- macro render_system_meta(tools) -%}\n {%- set rns = namespace(recipients=['\"self\"'], nslist=[]) -%}\n {%- if tools -%}\n {%- for tool in tools -%}\n {%- set fn = tool.function if tool.function is defined else tool -%}\n {%- set tns = fn.name.split('.')[0] -%}\n {%- if tns not in rns.nslist -%}\n {%- set rns.nslist = rns.nslist + [tns] -%}\n {%- endif -%}\n {%- endfor -%}\n {%- for tns in rns.nslist -%}\n {%- set rns.recipients = rns.recipients + ['\"' + tns + '.*\"'] -%}\n {%- endfor -%}\n {%- endif -%}\n {%- set rns.recipients = rns.recipients + ['\"user\"'] -%}\n {{- '# Valid recipients: ' + rns.recipients | join(', ') + '.' -}}\n{%- endmacro -%}\n{{- bos_token -}}\n{%- set ns = namespace(has_system=false) -%}\n{%- for m in messages -%}\n {%- if m['role'] == 'system' -%}\n {%- set ns.has_system = true -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if not ns.has_system -%}\n {{- '<|start|>system<|message|>You are a helpful AI assistant.' -}}\n {%- set kc = knowledge_cutoff if knowledge_cutoff is defined and knowledge_cutoff else '2026-01-04' -%}\n {{- '\\nKnowledge cutoff: ' + kc + '.' -}}\n {%- if current_date is defined and current_date -%}\n {{- '\\nCurrent date: ' + current_date + '.' -}}\n {%- elif strftime_now is defined -%}\n {{- '\\nCurrent date: ' + strftime_now('%Y-%m-%d') + '.' -}}\n {%- endif -%}\n {{- '\\n\\n' -}}\n {{- render_reasoning() -}}\n {%- if tools -%}\n {{- '\\n\\n' -}}\n {{- render_tool_defs(tools) -}}\n {%- endif -%}\n {{- '\\n\\n' -}}\n {{- render_system_meta(tools) -}}\n {{- '<|eot|>' -}}\n{%- endif -%}\n{%- for message in messages -%}\n {%- set role = message['role'] -%}\n {%- set end_token = '<|eom|>' if (not loop.last and messages[loop.index0 + 1]['role'] == role) else '<|eot|>' -%}\n {%- if role == 'system' -%}\n {#- Callers sometimes write the directive into the system prompt themselves.\n Normalise \"Reasoning effort\" to \"Reasoning strength\" (jinja has no\n case-insensitive replace, hence the four realistic casings), then skip\n the kwarg-driven line below if the prompt already carries one. -#}\n {%- set sys_text = render_content(message['content'])\n | replace('Reasoning effort', 'Reasoning strength')\n | replace('Reasoning Effort', 'Reasoning Strength')\n | replace('reasoning effort', 'reasoning strength')\n | replace('REASONING EFFORT', 'REASONING STRENGTH') -%}\n {{- '<|start|>system<|message|>' -}}\n {{- sys_text -}}\n {%- if 'reasoning strength' not in (sys_text | lower) -%}\n {{- '\\n\\n' -}}\n {{- render_reasoning() -}}\n {%- endif -%}\n {%- if tools -%}\n {{- '\\n\\n' -}}\n {{- render_tool_defs(tools) -}}\n {%- endif -%}\n {{- '\\n\\n' -}}\n {{- render_system_meta(tools) -}}\n {{- '<|eot|>' -}}\n {%- elif role == 'user' -%}\n {{- '<|start|>user<|message|>' -}}\n {{- render_content(message['content']) -}}\n {{- '<|eot|>' -}}\n {%- elif role == 'tool' -%}\n {%- set tname = message.get('name') -%}\n {%- if not tname -%}\n {%- set tcid = message.get('tool_call_id') -%}\n {%- set rns = namespace(name=tcid if tcid else '') -%}\n {%- for m in messages -%}\n {%- if m.get('tool_calls') -%}\n {%- for tc in m['tool_calls'] -%}\n {%- if tcid is not none and tc.id is defined and tc.id == tcid -%}\n {%- set rns.name = tc.function.name -%}\n {%- endif -%}\n {%- endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {%- set tname = rns.name -%}\n {%- endif -%}\n {{- '<|start|>tool ' + tname + '<|message|><tool_output name=\"' + tname + '\">\\n' -}}\n {{- render_content(message['content']) -}}\n {{- '\\n</tool_output><|eot|>' -}}\n {%- elif role == 'assistant' -%}\n {%- if message.get('reasoning_content') -%}\n {{- '<|start|>assistant to=self<|message|>' + message['reasoning_content'] + '<|eom|>' -}}\n {%- endif -%}\n {%- if message.get('tool_calls') -%}\n {%- for tc in message['tool_calls'] -%}\n {{- '<|start|>assistant to=' + tc.function.name + '<|message|>' -}}\n {{- render_atem(tc) -}}\n {%- if loop.last -%}\n {{- end_token -}}\n {%- else -%}\n {{- '<|eom|>' -}}\n {%- endif -%}\n {%- endfor -%}\n {%- else -%}\n {%- set recipient = message.get('recipient') or 'user' -%}\n {%- set end_turn = message.get('end_turn') -%}\n {%- if end_turn is none -%}\n {%- set end_turn = not (recipient and recipient != 'user') -%}\n {%- endif -%}\n {{- '<|start|>assistant' -}}\n {%- if recipient -%}\n {{- ' to=' + recipient -}}\n {%- endif -%}\n {{- '<|message|>' -}}\n {{- render_content(message['content']) -}}\n {{- ('<|eot|>' if end_turn else '<|eom|>') -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{- '<|start|>assistant' -}}\n{%- endif -%}\n",
"response_format": {
"reasoning_pattern": null,
"prefixes": [],
"turn_pattern": "\\s*to=(?:self<\\|message\\|>(?P<chain_of_thought>.*?)|user<\\|message\\|>(?P<response>.*?)|[^\\s<]+<\\|message\\|>(?P<tool_calls>.*?))(?:<\\|eom\\|><\\|start\\|>assistant|<\\|eot\\|>|<\\|end_of_text\\|>|\\Z)",
"tool_calls": {
"tags": [
"<atem:function_calls>",
"</atem:function_calls>"
],
"function_pattern": "<atem:invoke name=\"([^\"]+)\">(.*?)</atem:invoke>",
"parameter_pattern": "<atem:parameter name=\"([^\"]+)\">(.*?)</atem:parameter>"
}
},
"system_role_name": "system",
"user_role_name": "user",
"assistant_role_name": "assistant",
"eos_token": "<|end_of_text|>",
"bos_token": "<|begin_of_text|>",
"default_system_prompt": null,
"reasoning_config": {
"default_reasoning_effort": "high",
"field_name": "reasoning_strength",
"reasoning_effort_to_field_value": {
"low": "low",
"medium": "medium",
"high": "high",
"xhigh": "xhigh"
}
}
},
"decoder_config": {
"embedding_config": {
"type": "UntiedEmbeddingConfig",
"input_scale": null,
"logit_soft_cap": 20.0,
"logit_scale": 0.19611613513818404,
"permute_indices": false
},
"transformer_config": {
"residual_config": {
"type": "AdditiveResidualConfig"
},
"layer_configs": [
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": null,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": null
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": null,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": null
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": null,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": null
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": null,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": null
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": null,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": null
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": null,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": null
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": null,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": null
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": null,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": null
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": null,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": null
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": null,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": null
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": null,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": null
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": null,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": null
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": {
"type": "UnscaledRoPEConfig",
"base": 500000.0,
"max_sequence_length": 131072,
"head_dim": 128
}
},
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": null,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"mlp": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "DenseMLPConfig",
"linear_config": {},
"activation": {
"type": "SiLU",
"alpha": 1.0
},
"has_up_biases": false,
"has_down_biases": false,
"gate_clipping": null,
"up_clipping": null
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
}
},
"epilogue_configs": [],
"hidden_dim": null,
"kv_source_layer_index": null,
"rope_config": null
}
],
"output_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"model_dim": 6656,
"hidden_dim": 19968
},
"vocab_size": 202048,
"ple_model_config": null,
"embedding_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
}
},
"generation_config": {
"stop_token_ids": [
200001,
200008
],
"temperature": 1.0,
"top_k": 64,
"top_p": 0.95,
"min_p": null,
"banned_tokens": null,
"repetition_penalty": null,
"presence_penalty": null,
"frequency_penalty": null,
"suffix_repetition_length": null
}
}
{
"type": "LanguageModelConfig",
"token_codec_config": {
"type": "ChatCodecConfig",
"prompt_template": "{#\n Template: Muse Glimmer ATEM Chat Template\n Renders the ATEM tool-calling protocol: reasoning channel (to=self), tool\n channels (to=<tool>), and the user channel, plus tool definitions and the\n valid-recipient list in the system block.\n\n Whitespace note: every tag uses the {%- -%} / {{- -}} stripping markers, so\n the indentation below is purely for readability and contributes nothing to\n the rendered output.\n#}\n{%- macro render_content(content) -%}\n {%- if content is string -%}\n {{- content -}}\n {%- elif content is not none -%}\n {%- for part in content -%}\n {%- if part['type'] == 'image' -%}\n {{- '<|patch|>' -}}\n {%- elif part['type'] == 'video' -%}\n {{- '<|video|>' -}}\n {%- elif part['type'] == 'text' -%}\n {{- part['text'] -}}\n {%- endif -%}\n {%- endfor -%}\n {%- endif -%}\n{%- endmacro -%}\n{%- macro render_atem(tc) -%}\n {%- set args = tc.function.arguments -%}\n {%- if args is not mapping -%}\n {{- raise_exception('Muse Glimmer ATEM chat template requires tool_call.function.arguments to be a dict (mapping); a JSON string cannot be parsed in the HF jinja sandbox.') -}}\n {%- endif -%}\n {{- '<atem:function_calls>\\n<atem:invoke name=\"' + tc.function.name + '\">\\n' -}}\n {%- for k, v in args.items() -%}\n {{- '<atem:parameter name=\"' + k + '\">' -}}\n {%- if v is boolean -%}\n {%- if v -%}\n true\n {%- else -%}\n false\n {%- endif -%}\n {%- elif v is none -%}\n null\n {%- elif v is mapping or (v is iterable and v is not string) -%}\n {{- v | tojson -}}\n {%- else -%}\n {{- v -}}\n {%- endif -%}\n {{- '</atem:parameter>\\n' -}}\n {%- endfor -%}\n {{- '</atem:invoke>\\n</atem:function_calls>' -}}\n{%- endmacro -%}\n{%- macro render_tool_defs(tools) -%}\n {{- 'In this environment you have access to a set of tools you can use to answer the user\\'s question.\\n\\n' -}}\n {{- 'You can invoke a function by writing a \"<atem:function_calls>\" block like the following:\\n' -}}\n {{- '<atem:function_calls>\\n<atem:invoke name=\"$FUNCTION_NAME\">\\n<atem:parameter name=\"$PARAMETER_NAME\">$PARAMETER_VALUE</atem:parameter>\\n...\\n</atem:invoke>\\n</atem:function_calls>\\n\\n' -}}\n {{- 'String and scalar parameters should be specified as is, while lists and objects should use JSON format. Note that spaces for string values are not stripped. The output is not expected to be valid XML and is parsed with regular expressions.\\n' -}}\n {{- 'Here are the functions available in JSONSchema format:\\n' -}}\n {{- '// Tool metadata\\n' -}}\n {%- set nsns = namespace(seen=[]) -%}\n {%- for tool in tools -%}\n {%- set fn = tool.function if tool.function is defined else tool -%}\n {%- set tns = fn.name.split('.')[0] -%}\n {%- if tns not in nsns.seen -%}\n {%- set nsns.seen = nsns.seen + [tns] -%}\n {%- endif -%}\n {%- endfor -%}\n {%- set nd = tool_namespace_descriptions if tool_namespace_descriptions is defined else {} -%}\n {%- for tns in nsns.seen -%}\n {{- '{\"name\": ' + (tns | tojson) + ', \"description\": ' + ((nd[tns] if tns in nd else '') | tojson) + '}\\n' -}}\n {%- endfor -%}\n {{- '// Function schemas' -}}\n {%- for tool in tools -%}\n {%- set fn = tool.function if tool.function is defined else tool -%}\n {{- '\\n{\"name\": ' + (fn.name | tojson) + ', \"description\": ' + (fn.description | tojson) + ', \"parameters\": ' + (fn.parameters | tojson) + '}' -}}\n {%- endfor -%}\n {{- '\\n\\nHere\\'s an example of how to call a function in the tool set:\\n' -}}\n {{- '(If the tool namespace is not specified, invoke the function directly as `example_function_name` rather than `example_tool_name.example_function_name`)\\n\\n' -}}\n {{- 'to=example_tool_name.example_function_name\\n\\n' -}}\n {{- '<atem:function_calls>\\n<atem:invoke name=\"example_tool_name.example_function_name\">\\n' -}}\n {{- '<atem:parameter name=\"example_parameter_1\">value_1</atem:parameter>\\n' -}}\n {{- '<atem:parameter name=\"example_parameter_2\">This is the value for the second parameter\\nthat can span\\n\"multiple\" lines\\n</atem:parameter>\\n' -}}\n {{- '</atem:invoke>\\n</atem:function_calls>' -}}\n{%- endmacro -%}\n{%- macro render_reasoning() -%}\n {%- set rs = reasoning_strength if reasoning_strength is defined and reasoning_strength else 'high' -%}\n {{- 'Reasoning strength: ' + rs + '.' -}}\n{%- endmacro -%}\n{%- macro render_system_meta(tools) -%}\n {%- set rns = namespace(recipients=['\"self\"'], nslist=[]) -%}\n {%- if tools -%}\n {%- for tool in tools -%}\n {%- set fn = tool.function if tool.function is defined else tool -%}\n {%- set tns = fn.name.split('.')[0] -%}\n {%- if tns not in rns.nslist -%}\n {%- set rns.nslist = rns.nslist + [tns] -%}\n {%- endif -%}\n {%- endfor -%}\n {%- for tns in rns.nslist -%}\n {%- set rns.recipients = rns.recipients + ['\"' + tns + '.*\"'] -%}\n {%- endfor -%}\n {%- endif -%}\n {%- set rns.recipients = rns.recipients + ['\"user\"'] -%}\n {{- '# Valid recipients: ' + rns.recipients | join(', ') + '.' -}}\n{%- endmacro -%}\n{{- bos_token -}}\n{%- set ns = namespace(has_system=false) -%}\n{%- for m in messages -%}\n {%- if m['role'] == 'system' -%}\n {%- set ns.has_system = true -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if not ns.has_system -%}\n {{- '<|start|>system<|message|>You are a helpful AI assistant.' -}}\n {%- set kc = knowledge_cutoff if knowledge_cutoff is defined and knowledge_cutoff else '2026-01-04' -%}\n {{- '\\nKnowledge cutoff: ' + kc + '.' -}}\n {%- if current_date is defined and current_date -%}\n {{- '\\nCurrent date: ' + current_date + '.' -}}\n {%- elif strftime_now is defined -%}\n {{- '\\nCurrent date: ' + strftime_now('%Y-%m-%d') + '.' -}}\n {%- endif -%}\n {{- '\\n\\n' -}}\n {{- render_reasoning() -}}\n {%- if tools -%}\n {{- '\\n\\n' -}}\n {{- render_tool_defs(tools) -}}\n {%- endif -%}\n {{- '\\n\\n' -}}\n {{- render_system_meta(tools) -}}\n {{- '<|eot|>' -}}\n{%- endif -%}\n{%- for message in messages -%}\n {%- set role = message['role'] -%}\n {%- set end_token = '<|eom|>' if (not loop.last and messages[loop.index0 + 1]['role'] == role) else '<|eot|>' -%}\n {%- if role == 'system' -%}\n {#- Callers sometimes write the directive into the system prompt themselves.\n Normalise \"Reasoning effort\" to \"Reasoning strength\" (jinja has no\n case-insensitive replace, hence the four realistic casings), then skip\n the kwarg-driven line below if the prompt already carries one. -#}\n {%- set sys_text = render_content(message['content'])\n | replace('Reasoning effort', 'Reasoning strength')\n | replace('Reasoning Effort', 'Reasoning Strength')\n | replace('reasoning effort', 'reasoning strength')\n | replace('REASONING EFFORT', 'REASONING STRENGTH') -%}\n {{- '<|start|>system<|message|>' -}}\n {{- sys_text -}}\n {%- if 'reasoning strength' not in (sys_text | lower) -%}\n {{- '\\n\\n' -}}\n {{- render_reasoning() -}}\n {%- endif -%}\n {%- if tools -%}\n {{- '\\n\\n' -}}\n {{- render_tool_defs(tools) -}}\n {%- endif -%}\n {{- '\\n\\n' -}}\n {{- render_system_meta(tools) -}}\n {{- '<|eot|>' -}}\n {%- elif role == 'user' -%}\n {{- '<|start|>user<|message|>' -}}\n {{- render_content(message['content']) -}}\n {{- '<|eot|>' -}}\n {%- elif role == 'tool' -%}\n {%- set tname = message.get('name') -%}\n {%- if not tname -%}\n {%- set tcid = message.get('tool_call_id') -%}\n {%- set rns = namespace(name=tcid if tcid else '') -%}\n {%- for m in messages -%}\n {%- if m.get('tool_calls') -%}\n {%- for tc in m['tool_calls'] -%}\n {%- if tcid is not none and tc.id is defined and tc.id == tcid -%}\n {%- set rns.name = tc.function.name -%}\n {%- endif -%}\n {%- endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {%- set tname = rns.name -%}\n {%- endif -%}\n {{- '<|start|>tool ' + tname + '<|message|><tool_output name=\"' + tname + '\">\\n' -}}\n {{- render_content(message['content']) -}}\n {{- '\\n</tool_output><|eot|>' -}}\n {%- elif role == 'assistant' -%}\n {%- if message.get('reasoning_content') -%}\n {{- '<|start|>assistant to=self<|message|>' + message['reasoning_content'] + '<|eom|>' -}}\n {%- endif -%}\n {%- if message.get('tool_calls') -%}\n {%- for tc in message['tool_calls'] -%}\n {{- '<|start|>assistant to=' + tc.function.name + '<|message|>' -}}\n {{- render_atem(tc) -}}\n {%- if loop.last -%}\n {{- end_token -}}\n {%- else -%}\n {{- '<|eom|>' -}}\n {%- endif -%}\n {%- endfor -%}\n {%- else -%}\n {%- set recipient = message.get('recipient') or 'user' -%}\n {%- set end_turn = message.get('end_turn') -%}\n {%- if end_turn is none -%}\n {%- set end_turn = not (recipient and recipient != 'user') -%}\n {%- endif -%}\n {{- '<|start|>assistant' -}}\n {%- if recipient -%}\n {{- ' to=' + recipient -}}\n {%- endif -%}\n {{- '<|message|>' -}}\n {{- render_content(message['content']) -}}\n {{- ('<|eot|>' if end_turn else '<|eom|>') -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{- '<|start|>assistant' -}}\n{%- endif -%}\n",
"response_format": {
"reasoning_pattern": null,
"prefixes": [],
"turn_pattern": "\\s*to=(?:self<\\|message\\|>(?P<chain_of_thought>.*?)|user<\\|message\\|>(?P<response>.*?)|[^\\s<]+<\\|message\\|>(?P<tool_calls>.*?))(?:<\\|eom\\|><\\|start\\|>assistant|<\\|eot\\|>|<\\|end_of_text\\|>|\\Z)",
"tool_calls": {
"tags": [
"<atem:function_calls>",
"</atem:function_calls>"
],
"function_pattern": "<atem:invoke name=\"([^\"]+)\">(.*?)</atem:invoke>",
"parameter_pattern": "<atem:parameter name=\"([^\"]+)\">(.*?)</atem:parameter>"
}
},
"system_role_name": "system",
"user_role_name": "user",
"assistant_role_name": "assistant",
"eos_token": "<|end_of_text|>",
"bos_token": "<|begin_of_text|>",
"default_system_prompt": null,
"reasoning_config": {
"default_reasoning_effort": "high",
"field_name": "reasoning_strength",
"reasoning_effort_to_field_value": {
"low": "low",
"medium": "medium",
"high": "high",
"xhigh": "xhigh"
}
}
},
"decoder_config": {
"embedding_config": {
"type": "UntiedEmbeddingConfig",
"input_scale": null,
"logit_soft_cap": 20.0,
"logit_scale": 0.19611613513818404,
"permute_indices": false
},
"transformer_config": {
"residual_config": {
"type": "AdditiveResidualConfig"
},
"layer_configs": [
{
"prologue_configs": [],
"mixer": {
"pre_norm_config": {
"epsilon": 1e-05,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,
"has_biases": false
},
"conv_config": null,
"body_config": {
"type": "AttentionConfig",
"qkvg_projection_config": {},
"out_projection_config": {},
"query_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"key_norm_config": {
"epsilon": 1e-05,
"scale_offset": null,
"upcast_mode": "only_normalization",
"subtract_mean": false,
"has_scale": false,
"has_biases": false
},
"num_heads": 32,
"num_groups": 2,
"head_dim": 128,
"is_causal": true,
"scale": 0.3420629053989924,
"sliding_window_size": 2048,
"logit_soft_cap": null,
"has_sinks": false,
"has_qkvg_biases": false,
"has_out_biases": false,
"has_gate": true,
"normalize_values": false,
"is_kv_sharing": false
},
"post_norm_config": {
"epsilon": 1e-08,
"scale_offset": 1.0,
"upcast_mode": "full_layer",
"subtract_mean": false,
"has_scale": true,