{ "type": "LanguageModelConfig", "token_codec_config": { "type": "ChatCodecConfig", "prompt_template": "{#\n Template: Muse Glimmer ATEM Chat Template\n Renders the ATEM tool-calling protocol: reasoning channel (to=self), tool\n channels (to=), and the user channel, plus tool definitions and the\n valid-recipient list in the system block.\n\n Whitespace note: every tag uses the {%- -%} / {{- -}} stripping markers, so\n the indentation below is purely for readability and contributes nothing to\n the rendered output.\n#}\n{%- macro render_content(content) -%}\n {%- if content is string -%}\n {{- content -}}\n {%- elif content is not none -%}\n {%- for part in content -%}\n {%- if part['type'] == 'image' -%}\n {{- '<|patch|>' -}}\n {%- elif part['type'] == 'video' -%}\n {{- '<|video|>' -}}\n {%- elif part['type'] == 'text' -%}\n {{- part['text'] -}}\n {%- endif -%}\n {%- endfor -%}\n {%- endif -%}\n{%- endmacro -%}\n{%- macro render_atem(tc) -%}\n {%- set args = tc.function.arguments -%}\n {%- if args is not mapping -%}\n {{- raise_exception('Muse Glimmer ATEM chat template requires tool_call.function.arguments to be a dict (mapping); a JSON string cannot be parsed in the HF jinja sandbox.') -}}\n {%- endif -%}\n {{- '\\n\\n' -}}\n {%- for k, v in args.items() -%}\n {{- '' -}}\n {%- if v is boolean -%}\n {%- if v -%}\n true\n {%- else -%}\n false\n {%- endif -%}\n {%- elif v is none -%}\n null\n {%- elif v is mapping or (v is iterable and v is not string) -%}\n {{- v | tojson -}}\n {%- else -%}\n {{- v -}}\n {%- endif -%}\n {{- '\\n' -}}\n {%- endfor -%}\n {{- '\\n' -}}\n{%- endmacro -%}\n{%- macro render_tool_defs(tools) -%}\n {{- 'In this environment you have access to a set of tools you can use to answer the user\\'s question.\\n\\n' -}}\n {{- 'You can invoke a function by writing a \"\" block like the following:\\n' -}}\n {{- '\\n\\n$PARAMETER_VALUE\\n...\\n\\n\\n\\n' -}}\n {{- 'String and scalar parameters should be specified as is, while lists and objects should use JSON format. Note that spaces for string values are not stripped. The output is not expected to be valid XML and is parsed with regular expressions.\\n' -}}\n {{- 'Here are the functions available in JSONSchema format:\\n' -}}\n {{- '// Tool metadata\\n' -}}\n {%- set nsns = namespace(seen=[]) -%}\n {%- for tool in tools -%}\n {%- set fn = tool.function if tool.function is defined else tool -%}\n {%- set tns = fn.name.split('.')[0] -%}\n {%- if tns not in nsns.seen -%}\n {%- set nsns.seen = nsns.seen + [tns] -%}\n {%- endif -%}\n {%- endfor -%}\n {%- set nd = tool_namespace_descriptions if tool_namespace_descriptions is defined else {} -%}\n {%- for tns in nsns.seen -%}\n {{- '{\"name\": ' + (tns | tojson) + ', \"description\": ' + ((nd[tns] if tns in nd else '') | tojson) + '}\\n' -}}\n {%- endfor -%}\n {{- '// Function schemas' -}}\n {%- for tool in tools -%}\n {%- set fn = tool.function if tool.function is defined else tool -%}\n {{- '\\n{\"name\": ' + (fn.name | tojson) + ', \"description\": ' + (fn.description | tojson) + ', \"parameters\": ' + (fn.parameters | tojson) + '}' -}}\n {%- endfor -%}\n {{- '\\n\\nHere\\'s an example of how to call a function in the tool set:\\n' -}}\n {{- '(If the tool namespace is not specified, invoke the function directly as `example_function_name` rather than `example_tool_name.example_function_name`)\\n\\n' -}}\n {{- 'to=example_tool_name.example_function_name\\n\\n' -}}\n {{- '\\n\\n' -}}\n {{- 'value_1\\n' -}}\n {{- 'This is the value for the second parameter\\nthat can span\\n\"multiple\" lines\\n\\n' -}}\n {{- '\\n' -}}\n{%- endmacro -%}\n{%- macro render_reasoning() -%}\n {%- set rs = reasoning_strength if reasoning_strength is defined and reasoning_strength else 'high' -%}\n {{- 'Reasoning strength: ' + rs + '.' -}}\n{%- endmacro -%}\n{%- macro render_system_meta(tools) -%}\n {%- set rns = namespace(recipients=['\"self\"'], nslist=[]) -%}\n {%- if tools -%}\n {%- for tool in tools -%}\n {%- set fn = tool.function if tool.function is defined else tool -%}\n {%- set tns = fn.name.split('.')[0] -%}\n {%- if tns not in rns.nslist -%}\n {%- set rns.nslist = rns.nslist + [tns] -%}\n {%- endif -%}\n {%- endfor -%}\n {%- for tns in rns.nslist -%}\n {%- set rns.recipients = rns.recipients + ['\"' + tns + '.*\"'] -%}\n {%- endfor -%}\n {%- endif -%}\n {%- set rns.recipients = rns.recipients + ['\"user\"'] -%}\n {{- '# Valid recipients: ' + rns.recipients | join(', ') + '.' -}}\n{%- endmacro -%}\n{{- bos_token -}}\n{%- set ns = namespace(has_system=false) -%}\n{%- for m in messages -%}\n {%- if m['role'] == 'system' -%}\n {%- set ns.has_system = true -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if not ns.has_system -%}\n {{- '<|start|>system<|message|>You are a helpful AI assistant.' -}}\n {%- set kc = knowledge_cutoff if knowledge_cutoff is defined and knowledge_cutoff else '2026-01-04' -%}\n {{- '\\nKnowledge cutoff: ' + kc + '.' -}}\n {%- if current_date is defined and current_date -%}\n {{- '\\nCurrent date: ' + current_date + '.' -}}\n {%- elif strftime_now is defined -%}\n {{- '\\nCurrent date: ' + strftime_now('%Y-%m-%d') + '.' -}}\n {%- endif -%}\n {{- '\\n\\n' -}}\n {{- render_reasoning() -}}\n {%- if tools -%}\n {{- '\\n\\n' -}}\n {{- render_tool_defs(tools) -}}\n {%- endif -%}\n {{- '\\n\\n' -}}\n {{- render_system_meta(tools) -}}\n {{- '<|eot|>' -}}\n{%- endif -%}\n{%- for message in messages -%}\n {%- set role = message['role'] -%}\n {%- set end_token = '<|eom|>' if (not loop.last and messages[loop.index0 + 1]['role'] == role) else '<|eot|>' -%}\n {%- if role == 'system' -%}\n {#- Callers sometimes write the directive into the system prompt themselves.\n Normalise \"Reasoning effort\" to \"Reasoning strength\" (jinja has no\n case-insensitive replace, hence the four realistic casings), then skip\n the kwarg-driven line below if the prompt already carries one. -#}\n {%- set sys_text = render_content(message['content'])\n | replace('Reasoning effort', 'Reasoning strength')\n | replace('Reasoning Effort', 'Reasoning Strength')\n | replace('reasoning effort', 'reasoning strength')\n | replace('REASONING EFFORT', 'REASONING STRENGTH') -%}\n {{- '<|start|>system<|message|>' -}}\n {{- sys_text -}}\n {%- if 'reasoning strength' not in (sys_text | lower) -%}\n {{- '\\n\\n' -}}\n {{- render_reasoning() -}}\n {%- endif -%}\n {%- if tools -%}\n {{- '\\n\\n' -}}\n {{- render_tool_defs(tools) -}}\n {%- endif -%}\n {{- '\\n\\n' -}}\n {{- render_system_meta(tools) -}}\n {{- '<|eot|>' -}}\n {%- elif role == 'user' -%}\n {{- '<|start|>user<|message|>' -}}\n {{- render_content(message['content']) -}}\n {{- '<|eot|>' -}}\n {%- elif role == 'tool' -%}\n {%- set tname = message.get('name') -%}\n {%- if not tname -%}\n {%- set tcid = message.get('tool_call_id') -%}\n {%- set rns = namespace(name=tcid if tcid else '') -%}\n {%- for m in messages -%}\n {%- if m.get('tool_calls') -%}\n {%- for tc in m['tool_calls'] -%}\n {%- if tcid is not none and tc.id is defined and tc.id == tcid -%}\n {%- set rns.name = tc.function.name -%}\n {%- endif -%}\n {%- endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {%- set tname = rns.name -%}\n {%- endif -%}\n {{- '<|start|>tool ' + tname + '<|message|>\\n' -}}\n {{- render_content(message['content']) -}}\n {{- '\\n<|eot|>' -}}\n {%- elif role == 'assistant' -%}\n {%- if message.get('reasoning_content') -%}\n {{- '<|start|>assistant to=self<|message|>' + message['reasoning_content'] + '<|eom|>' -}}\n {%- endif -%}\n {%- if message.get('tool_calls') -%}\n {%- for tc in message['tool_calls'] -%}\n {{- '<|start|>assistant to=' + tc.function.name + '<|message|>' -}}\n {{- render_atem(tc) -}}\n {%- if loop.last -%}\n {{- end_token -}}\n {%- else -%}\n {{- '<|eom|>' -}}\n {%- endif -%}\n {%- endfor -%}\n {%- else -%}\n {%- set recipient = message.get('recipient') or 'user' -%}\n {%- set end_turn = message.get('end_turn') -%}\n {%- if end_turn is none -%}\n {%- set end_turn = not (recipient and recipient != 'user') -%}\n {%- endif -%}\n {{- '<|start|>assistant' -}}\n {%- if recipient -%}\n {{- ' to=' + recipient -}}\n {%- endif -%}\n {{- '<|message|>' -}}\n {{- render_content(message['content']) -}}\n {{- ('<|eot|>' if end_turn else '<|eom|>') -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{- '<|start|>assistant' -}}\n{%- endif -%}\n", "response_format": { "reasoning_pattern": null, "prefixes": [], "turn_pattern": "\\s*to=(?:self<\\|message\\|>(?P.*?)|user<\\|message\\|>(?P.*?)|[^\\s<]+<\\|message\\|>(?P.*?))(?:<\\|eom\\|><\\|start\\|>assistant|<\\|eot\\|>|<\\|end_of_text\\|>|\\Z)", "tool_calls": { "tags": [ "", "" ], "function_pattern": "(.*?)", "parameter_pattern": "(.*?)" } }, "system_role_name": "system", "user_role_name": "user", "assistant_role_name": "assistant", "eos_token": "<|end_of_text|>", "bos_token": "<|begin_of_text|>", "default_system_prompt": null, "reasoning_config": { "default_reasoning_effort": "high", "field_name": "reasoning_strength", "reasoning_effort_to_field_value": { "low": "low", "medium": "medium", "high": "high", "xhigh": "xhigh" } } }, "decoder_config": { "embedding_config": { "type": "UntiedEmbeddingConfig", "input_scale": null, "logit_soft_cap": 20.0, "logit_scale": 0.19611613513818404, "permute_indices": false }, "transformer_config": { "residual_config": { "type": "AdditiveResidualConfig" }, "layer_configs": [ { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": null, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": null }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": null, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": null }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": null, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": null }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": null, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": null }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": null, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": null }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": null, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": null }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": null, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": null }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": null, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": null }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": null, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": null }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": null, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": null }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": null, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": null }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": null, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": null }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": 2048, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": { "type": "UnscaledRoPEConfig", "base": 500000.0, "max_sequence_length": 131072, "head_dim": 128 } }, { "prologue_configs": [], "mixer": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {}, "query_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "key_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false }, "num_heads": 32, "num_groups": 2, "head_dim": 128, "is_causal": true, "scale": 0.3420629053989924, "sliding_window_size": null, "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false, "has_out_biases": false, "has_gate": true, "normalize_values": false, "is_kv_sharing": false }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "mlp": { "pre_norm_config": { "epsilon": 1e-05, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "conv_config": null, "body_config": { "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 }, "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null }, "post_norm_config": { "epsilon": 1e-08, "scale_offset": 1.0, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false } }, "epilogue_configs": [], "hidden_dim": null, "kv_source_layer_index": null, "rope_config": null } ], "output_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "full_layer", "subtract_mean": false, "has_scale": true, "has_biases": false }, "model_dim": 6656, "hidden_dim": 19968 }, "vocab_size": 202048, "ple_model_config": null, "embedding_norm_config": { "epsilon": 1e-05, "scale_offset": null, "upcast_mode": "only_normalization", "subtract_mean": false, "has_scale": false, "has_biases": false } }, "generation_config": { "stop_token_ids": [ 200001, 200008 ], "temperature": 1.0, "top_k": 64, "top_p": 0.95, "min_p": null, "banned_tokens": null, "repetition_penalty": null, "presence_penalty": null, "frequency_penalty": null, "suffix_repetition_length": null } }