Sync tokenizer files from google/gemma-3-4b-it
Browse files- added_tokens.json +1 -0
- chat_template.jinja +47 -0
- chat_template.json +3 -0
- preprocessor_config.json +29 -0
- processor_config.json +4 -0
- special_tokens_map.json +33 -0
- tokenizer.json +0 -0
- tokenizer.model +3 -0
- tokenizer_config.json +1 -0
added_tokens.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"<image_soft_token>": 131072}
|
chat_template.jinja
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{{ bos_token }}
|
| 2 |
+
{%- if messages[0]['role'] == 'system' -%}
|
| 3 |
+
{%- if messages[0]['content'] is string -%}
|
| 4 |
+
{%- set first_user_prefix = messages[0]['content'] + '
|
| 5 |
+
|
| 6 |
+
' -%}
|
| 7 |
+
{%- else -%}
|
| 8 |
+
{%- set first_user_prefix = messages[0]['content'][0]['text'] + '
|
| 9 |
+
|
| 10 |
+
' -%}
|
| 11 |
+
{%- endif -%}
|
| 12 |
+
{%- set loop_messages = messages[1:] -%}
|
| 13 |
+
{%- else -%}
|
| 14 |
+
{%- set first_user_prefix = "" -%}
|
| 15 |
+
{%- set loop_messages = messages -%}
|
| 16 |
+
{%- endif -%}
|
| 17 |
+
{%- for message in loop_messages -%}
|
| 18 |
+
{%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
|
| 19 |
+
{{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
|
| 20 |
+
{%- endif -%}
|
| 21 |
+
{%- if (message['role'] == 'assistant') -%}
|
| 22 |
+
{%- set role = "model" -%}
|
| 23 |
+
{%- else -%}
|
| 24 |
+
{%- set role = message['role'] -%}
|
| 25 |
+
{%- endif -%}
|
| 26 |
+
{{ '<start_of_turn>' + role + '
|
| 27 |
+
' + (first_user_prefix if loop.first else "") }}
|
| 28 |
+
{%- if message['content'] is string -%}
|
| 29 |
+
{{ message['content'] | trim }}
|
| 30 |
+
{%- elif message['content'] is iterable -%}
|
| 31 |
+
{%- for item in message['content'] -%}
|
| 32 |
+
{%- if item['type'] == 'image' -%}
|
| 33 |
+
{{ '<start_of_image>' }}
|
| 34 |
+
{%- elif item['type'] == 'text' -%}
|
| 35 |
+
{{ item['text'] | trim }}
|
| 36 |
+
{%- endif -%}
|
| 37 |
+
{%- endfor -%}
|
| 38 |
+
{%- else -%}
|
| 39 |
+
{{ raise_exception("Invalid content type") }}
|
| 40 |
+
{%- endif -%}
|
| 41 |
+
{{ '<end_of_turn>
|
| 42 |
+
' }}
|
| 43 |
+
{%- endfor -%}
|
| 44 |
+
{%- if add_generation_prompt -%}
|
| 45 |
+
{{'<start_of_turn>model
|
| 46 |
+
'}}
|
| 47 |
+
{%- endif -%}
|
chat_template.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"chat_template": "{{ bos_token }}\n{%- if messages[0]['role'] == 'system' -%}\n {%- if messages[0]['content'] is string -%}\n {%- set first_user_prefix = messages[0]['content'] + '\n\n' -%}\n {%- else -%}\n {%- set first_user_prefix = messages[0]['content'][0]['text'] + '\n\n' -%}\n {%- endif -%}\n {%- set loop_messages = messages[1:] -%}\n{%- else -%}\n {%- set first_user_prefix = \"\" -%}\n {%- set loop_messages = messages -%}\n{%- endif -%}\n{%- for message in loop_messages -%}\n {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}\n {{ raise_exception(\"Conversation roles must alternate user/assistant/user/assistant/...\") }}\n {%- endif -%}\n {%- if (message['role'] == 'assistant') -%}\n {%- set role = \"model\" -%}\n {%- else -%}\n {%- set role = message['role'] -%}\n {%- endif -%}\n {{ '<start_of_turn>' + role + '\n' + (first_user_prefix if loop.first else \"\") }}\n {%- if message['content'] is string -%}\n {{ message['content'] | trim }}\n {%- elif message['content'] is iterable -%}\n {%- for item in message['content'] -%}\n {%- if item['type'] == 'image' -%}\n {{ '<start_of_image>' }}\n {%- elif item['type'] == 'text' -%}\n {{ item['text'] | trim }}\n {%- endif -%}\n {%- endfor -%}\n {%- else -%}\n {{ raise_exception(\"Invalid content type\") }}\n {%- endif -%}\n {{ '<end_of_turn>\n' }}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{'<start_of_turn>model\n'}}\n{%- endif -%}\n"
|
| 3 |
+
}
|
preprocessor_config.json
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"do_convert_rgb": null,
|
| 3 |
+
"do_normalize": true,
|
| 4 |
+
"do_pan_and_scan": null,
|
| 5 |
+
"do_rescale": true,
|
| 6 |
+
"do_resize": true,
|
| 7 |
+
"image_mean": [
|
| 8 |
+
0.5,
|
| 9 |
+
0.5,
|
| 10 |
+
0.5
|
| 11 |
+
],
|
| 12 |
+
"image_processor_type": "Gemma3ImageProcessor",
|
| 13 |
+
"image_seq_length": 256,
|
| 14 |
+
"image_std": [
|
| 15 |
+
0.5,
|
| 16 |
+
0.5,
|
| 17 |
+
0.5
|
| 18 |
+
],
|
| 19 |
+
"pan_and_scan_max_num_crops": null,
|
| 20 |
+
"pan_and_scan_min_crop_size": null,
|
| 21 |
+
"pan_and_scan_min_ratio_to_activate": null,
|
| 22 |
+
"processor_class": "Gemma3Processor",
|
| 23 |
+
"resample": 2,
|
| 24 |
+
"rescale_factor": 0.00392156862745098,
|
| 25 |
+
"size": {
|
| 26 |
+
"height": 896,
|
| 27 |
+
"width": 896
|
| 28 |
+
}
|
| 29 |
+
}
|
processor_config.json
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"image_seq_length": 256,
|
| 3 |
+
"processor_class": "Gemma3Processor"
|
| 4 |
+
}
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"boi_token": "<start_of_image>",
|
| 3 |
+
"bos_token": {
|
| 4 |
+
"content": "<bos>",
|
| 5 |
+
"lstrip": false,
|
| 6 |
+
"normalized": false,
|
| 7 |
+
"rstrip": false,
|
| 8 |
+
"single_word": false
|
| 9 |
+
},
|
| 10 |
+
"eoi_token": "<end_of_image>",
|
| 11 |
+
"eos_token": {
|
| 12 |
+
"content": "<eos>",
|
| 13 |
+
"lstrip": false,
|
| 14 |
+
"normalized": false,
|
| 15 |
+
"rstrip": false,
|
| 16 |
+
"single_word": false
|
| 17 |
+
},
|
| 18 |
+
"image_token": "<image_soft_token>",
|
| 19 |
+
"pad_token": {
|
| 20 |
+
"content": "<pad>",
|
| 21 |
+
"lstrip": false,
|
| 22 |
+
"normalized": false,
|
| 23 |
+
"rstrip": false,
|
| 24 |
+
"single_word": false
|
| 25 |
+
},
|
| 26 |
+
"unk_token": {
|
| 27 |
+
"content": "<unk>",
|
| 28 |
+
"lstrip": false,
|
| 29 |
+
"normalized": false,
|
| 30 |
+
"rstrip": false,
|
| 31 |
+
"single_word": false
|
| 32 |
+
}
|
| 33 |
+
}
|
tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer.model
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ede2bd20a814c5298d01d10b4586c9f187bf938ce7be99fedead9caefa2b0795
|
| 3 |
+
size 2112925
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"add_bos_token": true, "add_eos_token": false, "added_tokens_decoder": {"0": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "1": {"content": "<eos>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "2": {"content": "<bos>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "3": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "4": {"content": "<mask>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "5": {"content": "[multimodal]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "6": {"content": "<unused0>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "7": {"content": "<unused1>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "8": {"content": "<unused2>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "9": {"content": "<unused3>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "10": {"content": "<unused4>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "11": {"content": "<unused5>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "12": {"content": "<unused6>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "13": {"content": "<unused7>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "14": {"content": "<unused8>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "15": {"content": "<unused9>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "16": {"content": "<unused10>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "17": {"content": "<unused11>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "18": {"content": "<unused12>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "19": {"content": "<unused13>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "20": {"content": "<unused14>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "21": {"content": "<unused15>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "22": {"content": "<unused16>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "23": {"content": "<unused17>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "24": {"content": "<unused18>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "25": {"content": "<unused19>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "26": {"content": "<unused20>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "27": {"content": "<unused21>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "28": {"content": "<unused22>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "29": {"content": "<unused23>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "30": {"content": "<unused24>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "31": {"content": "<unused25>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32": {"content": "<unused26>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "33": {"content": "<unused27>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "34": {"content": "<unused28>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "35": {"content": "<unused29>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "36": {"content": "<unused30>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "37": {"content": "<unused31>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "38": {"content": "<unused32>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "39": {"content": "<unused33>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "40": {"content": "<unused34>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "41": {"content": "<unused35>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "42": {"content": "<unused36>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "43": {"content": "<unused37>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "44": {"content": "<unused38>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "45": {"content": "<unused39>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "46": {"content": "<unused40>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "47": {"content": "<unused41>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "48": {"content": "<unused42>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "49": {"content": "<unused43>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "50": {"content": "<unused44>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "51": {"content": "<unused45>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "52": {"content": "<unused46>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "53": {"content": "<unused47>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "54": {"content": "<unused48>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "55": {"content": "<unused49>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "56": {"content": "<unused50>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "57": {"content": "<unused51>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "58": {"content": "<unused52>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "59": {"content": "<unused53>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "60": {"content": "<unused54>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "61": {"content": "<unused55>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "62": {"content": "<unused56>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "63": {"content": "<unused57>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "64": {"content": "<unused58>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "65": {"content": "<unused59>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "66": {"content": "<unused60>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "67": {"content": "<unused61>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "68": {"content": "<unused62>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "69": {"content": "<unused63>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "70": {"content": "<unused64>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "71": {"content": "<unused65>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "72": {"content": "<unused66>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "73": {"content": "<unused67>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "74": {"content": "<unused68>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "75": {"content": "<unused69>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "76": {"content": "<unused70>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "77": {"content": "<unused71>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "78": {"content": "<unused72>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "79": {"content": "<unused73>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "80": {"content": "<unused74>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "81": {"content": "<unused75>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "82": {"content": "<unused76>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "83": {"content": "<unused77>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "84": {"content": "<unused78>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "85": {"content": "<unused79>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "86": {"content": "<unused80>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "87": {"content": "<unused81>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "88": {"content": "<unused82>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "89": {"content": "<unused83>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "90": {"content": "<unused84>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "91": {"content": "<unused85>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "92": {"content": "<unused86>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "93": {"content": "<unused87>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "94": {"content": "<unused88>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "95": {"content": "<unused89>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "96": {"content": "<unused90>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "97": {"content": "<unused91>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "98": {"content": "<unused92>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "99": {"content": "<unused93>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "100": {"content": "<unused94>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "101": {"content": "<unused95>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "102": {"content": "<unused96>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "103": {"content": "<unused97>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "104": {"content": "<unused98>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "105": {"content": "<unused99>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "106": {"content": "<start_of_turn>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "107": {"content": "<end_of_turn>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "112": {"content": "<table>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "113": {"content": "<caption>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "114": {"content": "<thead>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "115": {"content": "<tbody>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "116": {"content": "<tfoot>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "117": {"content": "<tr>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "118": {"content": "<th>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "119": {"content": "<td>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "120": {"content": "</table>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "121": {"content": "</caption>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "122": {"content": "</thead>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "123": {"content": "</tbody>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "124": {"content": "</tfoot>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "125": {"content": "</tr>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "126": {"content": "</th>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "127": {"content": "</td>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "128": {"content": "<h1>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "129": {"content": "<h2>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130": {"content": "<h3>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131": {"content": "<h4>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "132": {"content": "<h5>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "133": {"content": "<h6>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "134": {"content": "<blockquote>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "135": {"content": "</h1>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "136": {"content": "</h2>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "137": {"content": "</h3>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "138": {"content": "</h4>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "139": {"content": "</h5>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "140": {"content": "</h6>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "141": {"content": "</blockquote>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "142": {"content": "<strong>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "143": {"content": "<em>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "144": {"content": "<b>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "145": {"content": "<i>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "146": {"content": "<u>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "147": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "148": {"content": "<sub>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "149": {"content": "<sup>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "150": {"content": "<code>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "151": {"content": "</strong>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "152": {"content": "</em>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "153": {"content": "</b>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "154": {"content": "</i>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "155": {"content": "</u>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "156": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "157": {"content": "</sub>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "158": {"content": "</sup>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "159": {"content": "</code>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "160": {"content": "<a>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "161": {"content": "<html>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "162": {"content": "<body>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "163": {"content": "<img>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "164": {"content": "<span>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "165": {"content": "<bbox>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "166": {"content": "<ul>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "167": {"content": "<li>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "168": {"content": "<div>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "169": {"content": "<iframe>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "170": {"content": "<footer>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "171": {"content": "</a>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "172": {"content": "</html>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "173": {"content": "</body>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "174": {"content": "</img>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "175": {"content": "</span>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "176": {"content": "</bbox>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "177": {"content": "</ul>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "178": {"content": "</li>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "179": {"content": "</div>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "180": {"content": "</iframe>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "181": {"content": "</footer>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "28850": {"content": "<>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32027": {"content": "<?>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "68027": {"content": "<=>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130970": {"content": "<start_of_image>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130971": {"content": "<end_of_image>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130972": {"content": "<unused101>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130973": {"content": "<unused102>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130974": {"content": "<unused103>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130975": {"content": "<unused104>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130976": {"content": "<unused105>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130977": {"content": "<unused106>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130978": {"content": "<unused107>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130979": {"content": "<unused108>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130980": {"content": "<unused109>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130981": {"content": "<unused110>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130982": {"content": "<unused111>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130983": {"content": "<unused112>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130984": {"content": "<unused113>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130985": {"content": "<unused114>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130986": {"content": "<unused115>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130987": {"content": "<unused116>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130988": {"content": "<unused117>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130989": {"content": "<unused118>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130990": {"content": "<unused119>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130991": {"content": "<unused120>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130992": {"content": "<unused121>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130993": {"content": "<unused122>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130994": {"content": "<unused123>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130995": {"content": "<unused124>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130996": {"content": "<unused125>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130997": {"content": "<unused126>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130998": {"content": "<unused127>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "130999": {"content": "<unused128>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131000": {"content": "<unused129>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131001": {"content": "<unused130>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131002": {"content": "<unused131>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131003": {"content": "<unused132>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131004": {"content": "<unused133>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131005": {"content": "<unused134>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131006": {"content": "<unused135>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131007": {"content": "<unused136>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131008": {"content": "<unused137>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131009": {"content": "<unused138>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131010": {"content": "<unused139>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131011": {"content": "<unused140>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131012": {"content": "<unused141>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131013": {"content": "<unused142>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131014": {"content": "<unused143>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131015": {"content": "<unused144>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131016": {"content": "<unused145>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131017": {"content": "<unused146>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131018": {"content": "<unused147>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131019": {"content": "<unused148>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131020": {"content": "<unused149>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131021": {"content": "<unused150>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131022": {"content": "<unused151>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131023": {"content": "<unused152>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131024": {"content": "<unused153>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131025": {"content": "<unused154>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131026": {"content": "<unused155>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131027": {"content": "<unused156>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131028": {"content": "<unused157>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131029": {"content": "<unused158>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131030": {"content": "<unused159>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131031": {"content": "<unused160>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131032": {"content": "<unused161>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131033": {"content": "<unused162>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131034": {"content": "<unused163>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131035": {"content": "<unused164>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131036": {"content": "<unused165>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131037": {"content": "<unused166>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131038": {"content": "<unused167>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131039": {"content": "<unused168>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131040": {"content": "<unused169>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131041": {"content": "<unused170>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131042": {"content": "<unused171>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131043": {"content": "<unused172>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131044": {"content": "<unused173>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131045": {"content": "<unused174>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131046": {"content": "<unused175>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131047": {"content": "<unused176>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131048": {"content": "<unused177>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131049": {"content": "<unused178>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131050": {"content": "<unused179>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131051": {"content": "<unused180>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131052": {"content": "<unused181>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131053": {"content": "<unused182>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131054": {"content": "<unused183>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131055": {"content": "<unused184>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131056": {"content": "<unused185>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131057": {"content": "<unused186>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131058": {"content": "<unused187>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131059": {"content": "<unused188>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131060": {"content": "<unused189>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131061": {"content": "<unused190>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131062": {"content": "<unused191>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131063": {"content": "<unused192>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131064": {"content": "<unused193>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131065": {"content": "<unused194>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131066": {"content": "<unused195>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131067": {"content": "<unused196>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131068": {"content": "<unused197>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131069": {"content": "<unused198>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131070": {"content": "<unused199>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131071": {"content": "<unused200>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "131072": {"content": "<image_soft_token>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true}}, "boi_token": "<start_of_image>", "bos_token": "<bos>", "chat_template": "{{ bos_token }}\n{%- if messages[0]['role'] == 'system' -%}\n {%- if messages[0]['content'] is string -%}\n {%- set first_user_prefix = messages[0]['content'] + '\n\n' -%}\n {%- else -%}\n {%- set first_user_prefix = messages[0]['content'][0]['text'] + '\n\n' -%}\n {%- endif -%}\n {%- set loop_messages = messages[1:] -%}\n{%- else -%}\n {%- set first_user_prefix = \"\" -%}\n {%- set loop_messages = messages -%}\n{%- endif -%}\n{%- for message in loop_messages -%}\n {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}\n {{ raise_exception(\"Conversation roles must alternate user/assistant/user/assistant/...\") }}\n {%- endif -%}\n {%- if (message['role'] == 'assistant') -%}\n {%- set role = \"model\" -%}\n {%- else -%}\n {%- set role = message['role'] -%}\n {%- endif -%}\n {{ '<start_of_turn>' + role + '\n' + (first_user_prefix if loop.first else \"\") }}\n {%- if message['content'] is string -%}\n {{ message['content'] | trim }}\n {%- elif message['content'] is iterable -%}\n {%- for item in message['content'] -%}\n {%- if item['type'] == 'image' -%}\n {{ '<start_of_image>' }}\n {%- elif item['type'] == 'text' -%}\n {{ item['text'] | trim }}\n {%- endif -%}\n {%- endfor -%}\n {%- else -%}\n {{ raise_exception(\"Invalid content type\") }}\n {%- endif -%}\n {{ '<end_of_turn>\n' }}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{'<start_of_turn>model\n'}}\n{%- endif -%}\n", "clean_up_tokenization_spaces": false, "eoi_token": "<end_of_image>", "eos_token": "<eos>", "extra_special_tokens": {"boi_token": "<start_of_image>", "eoi_token": "<end_of_image>", "image_token": "<image_soft_token>"}, "image_token": "<image_soft_token>", "model_max_length": 1000000000000000019884624838656, "pad_token": "<pad>", "processor_class": "Gemma3Processor", "sp_model_kwargs": null, "spaces_between_special_tokens": false, "tokenizer_class": "GemmaTokenizer", "unk_token": "<unk>", "use_default_system_prompt": false, "vocab_file": "tokenizer.model"}
|