Hugging Face's logo

mlboydaisuke
/
decider-2b-coreai-ft

Text Classification
Transformers
Safetensors
English
qwen3_5_text
text-generation
decision-model
system-one
typed-decisions
calibrated
decider
Model card Files
xet
 Community
mlboydaisuke commited on
Commit
86616d8
·
verified ·
1 Parent(s): 436153d

decider-2b v11 + 1,399 verified typed-decision items (delta recipe, 3 epochs, lr 8e-6)

Browse files
Files changed (12) hide show
  1. .gitattributes +1 -0
  2. chat_template.jinja +154 -0
  3. config.json +75 -0
  4. decider_config.json +21 -0
  5. eval/hist.json +35 -0
  6. eval/jevbench_public231_baseline_decider-2b.json +3646 -0
  7. eval/jevbench_public231_this_model.json +3646 -0
  8. eval/train.log +23 -0
  9. generation_config.json +6 -0
  10. model.safetensors +3 -0
  11. tokenizer.json +3 -0
  12. tokenizer_config.json +32 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
chat_template.jinja ADDED
@@ -0,0 +1,154 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- set image_count = namespace(value=0) %}
2
+ {%- set video_count = namespace(value=0) %}
3
+ {%- macro render_content(content, do_vision_count, is_system_content=false) %}
4
+ {%- if content is string %}
5
+ {{- content }}
6
+ {%- elif content is iterable and content is not mapping %}
7
+ {%- for item in content %}
8
+ {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
9
+ {%- if is_system_content %}
10
+ {{- raise_exception('System message cannot contain images.') }}
11
+ {%- endif %}
12
+ {%- if do_vision_count %}
13
+ {%- set image_count.value = image_count.value + 1 %}
14
+ {%- endif %}
15
+ {%- if add_vision_id %}
16
+ {{- 'Picture ' ~ image_count.value ~ ': ' }}
17
+ {%- endif %}
18
+ {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
19
+ {%- elif 'video' in item or item.type == 'video' %}
20
+ {%- if is_system_content %}
21
+ {{- raise_exception('System message cannot contain videos.') }}
22
+ {%- endif %}
23
+ {%- if do_vision_count %}
24
+ {%- set video_count.value = video_count.value + 1 %}
25
+ {%- endif %}
26
+ {%- if add_vision_id %}
27
+ {{- 'Video ' ~ video_count.value ~ ': ' }}
28
+ {%- endif %}
29
+ {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
30
+ {%- elif 'text' in item %}
31
+ {{- item.text }}
32
+ {%- else %}
33
+ {{- raise_exception('Unexpected item type in content.') }}
34
+ {%- endif %}
35
+ {%- endfor %}
36
+ {%- elif content is none or content is undefined %}
37
+ {{- '' }}
38
+ {%- else %}
39
+ {{- raise_exception('Unexpected content type.') }}
40
+ {%- endif %}
41
+ {%- endmacro %}
42
+ {%- if not messages %}
43
+ {{- raise_exception('No messages provided.') }}
44
+ {%- endif %}
45
+ {%- if tools and tools is iterable and tools is not mapping %}
46
+ {{- '<|im_start|>system\n' }}
47
+ {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
48
+ {%- for tool in tools %}
49
+ {{- "\n" }}
50
+ {{- tool | tojson }}
51
+ {%- endfor %}
52
+ {{- "\n</tools>" }}
53
+ {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
54
+ {%- if messages[0].role == 'system' %}
55
+ {%- set content = render_content(messages[0].content, false, true)|trim %}
56
+ {%- if content %}
57
+ {{- '\n\n' + content }}
58
+ {%- endif %}
59
+ {%- endif %}
60
+ {{- '<|im_end|>\n' }}
61
+ {%- else %}
62
+ {%- if messages[0].role == 'system' %}
63
+ {%- set content = render_content(messages[0].content, false, true)|trim %}
64
+ {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
65
+ {%- endif %}
66
+ {%- endif %}
67
+ {%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
68
+ {%- for message in messages[::-1] %}
69
+ {%- set index = (messages|length - 1) - loop.index0 %}
70
+ {%- if ns.multi_step_tool and message.role == "user" %}
71
+ {%- set content = render_content(message.content, false)|trim %}
72
+ {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
73
+ {%- set ns.multi_step_tool = false %}
74
+ {%- set ns.last_query_index = index %}
75
+ {%- endif %}
76
+ {%- endif %}
77
+ {%- endfor %}
78
+ {%- if ns.multi_step_tool %}
79
+ {{- raise_exception('No user query found in messages.') }}
80
+ {%- endif %}
81
+ {%- for message in messages %}
82
+ {%- set content = render_content(message.content, true)|trim %}
83
+ {%- if message.role == "system" %}
84
+ {%- if not loop.first %}
85
+ {{- raise_exception('System message must be at the beginning.') }}
86
+ {%- endif %}
87
+ {%- elif message.role == "user" %}
88
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
89
+ {%- elif message.role == "assistant" %}
90
+ {%- set reasoning_content = '' %}
91
+ {%- if message.reasoning_content is string %}
92
+ {%- set reasoning_content = message.reasoning_content %}
93
+ {%- else %}
94
+ {%- if '</think>' in content %}
95
+ {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
96
+ {%- set content = content.split('</think>')[-1].lstrip('\n') %}
97
+ {%- endif %}
98
+ {%- endif %}
99
+ {%- set reasoning_content = reasoning_content|trim %}
100
+ {%- if loop.index0 > ns.last_query_index %}
101
+ {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
102
+ {%- else %}
103
+ {{- '<|im_start|>' + message.role + '\n' + content }}
104
+ {%- endif %}
105
+ {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
106
+ {%- for tool_call in message.tool_calls %}
107
+ {%- if tool_call.function is defined %}
108
+ {%- set tool_call = tool_call.function %}
109
+ {%- endif %}
110
+ {%- if loop.first %}
111
+ {%- if content|trim %}
112
+ {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
113
+ {%- else %}
114
+ {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
115
+ {%- endif %}
116
+ {%- else %}
117
+ {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
118
+ {%- endif %}
119
+ {%- if tool_call.arguments is defined %}
120
+ {%- for args_name, args_value in tool_call.arguments|items %}
121
+ {{- '<parameter=' + args_name + '>\n' }}
122
+ {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
123
+ {{- args_value }}
124
+ {{- '\n</parameter>\n' }}
125
+ {%- endfor %}
126
+ {%- endif %}
127
+ {{- '</function>\n</tool_call>' }}
128
+ {%- endfor %}
129
+ {%- endif %}
130
+ {{- '<|im_end|>\n' }}
131
+ {%- elif message.role == "tool" %}
132
+ {%- if loop.previtem and loop.previtem.role != "tool" %}
133
+ {{- '<|im_start|>user' }}
134
+ {%- endif %}
135
+ {{- '\n<tool_response>\n' }}
136
+ {{- content }}
137
+ {{- '\n</tool_response>' }}
138
+ {%- if not loop.last and loop.nextitem.role != "tool" %}
139
+ {{- '<|im_end|>\n' }}
140
+ {%- elif loop.last %}
141
+ {{- '<|im_end|>\n' }}
142
+ {%- endif %}
143
+ {%- else %}
144
+ {{- raise_exception('Unexpected message role.') }}
145
+ {%- endif %}
146
+ {%- endfor %}
147
+ {%- if add_generation_prompt %}
148
+ {{- '<|im_start|>assistant\n' }}
149
+ {%- if enable_thinking is defined and enable_thinking is true %}
150
+ {{- '<think>\n' }}
151
+ {%- else %}
152
+ {{- '<think>\n\n</think>\n\n' }}
153
+ {%- endif %}
154
+ {%- endif %}
config.json ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3_5ForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "attn_output_gate": true,
8
+ "bos_token_id": null,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 248044,
11
+ "full_attention_interval": 4,
12
+ "head_dim": 256,
13
+ "hidden_act": "silu",
14
+ "hidden_size": 2048,
15
+ "initializer_range": 0.02,
16
+ "intermediate_size": 6144,
17
+ "layer_types": [
18
+ "linear_attention",
19
+ "linear_attention",
20
+ "linear_attention",
21
+ "full_attention",
22
+ "linear_attention",
23
+ "linear_attention",
24
+ "linear_attention",
25
+ "full_attention",
26
+ "linear_attention",
27
+ "linear_attention",
28
+ "linear_attention",
29
+ "full_attention",
30
+ "linear_attention",
31
+ "linear_attention",
32
+ "linear_attention",
33
+ "full_attention",
34
+ "linear_attention",
35
+ "linear_attention",
36
+ "linear_attention",
37
+ "full_attention",
38
+ "linear_attention",
39
+ "linear_attention",
40
+ "linear_attention",
41
+ "full_attention"
42
+ ],
43
+ "linear_conv_kernel_dim": 4,
44
+ "linear_key_head_dim": 128,
45
+ "linear_num_key_heads": 16,
46
+ "linear_num_value_heads": 16,
47
+ "linear_value_head_dim": 128,
48
+ "mamba_ssm_dtype": "float32",
49
+ "max_position_embeddings": 262144,
50
+ "mlp_only_layers": [],
51
+ "model_type": "qwen3_5_text",
52
+ "mtp_num_hidden_layers": 1,
53
+ "mtp_use_dedicated_embeddings": false,
54
+ "num_attention_heads": 8,
55
+ "num_hidden_layers": 24,
56
+ "num_key_value_heads": 2,
57
+ "pad_token_id": null,
58
+ "partial_rotary_factor": 0.25,
59
+ "rms_norm_eps": 1e-06,
60
+ "rope_parameters": {
61
+ "mrope_interleaved": true,
62
+ "mrope_section": [
63
+ 11,
64
+ 11,
65
+ 10
66
+ ],
67
+ "partial_rotary_factor": 0.25,
68
+ "rope_theta": 10000000,
69
+ "rope_type": "default"
70
+ },
71
+ "tie_word_embeddings": true,
72
+ "transformers_version": "5.16.1",
73
+ "use_cache": true,
74
+ "vocab_size": 248320
75
+ }
decider_config.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "temperature": 1.145,
3
+ "temperature_by_type": {
4
+ "choice": 1.164,
5
+ "noul": 1.624,
6
+ "score": 1.124
7
+ },
8
+ "neutralize_none": false,
9
+ "version": "2b-v11",
10
+ "base": "Mapika/decider-2b v10 + LoRA (merged); v10 is Qwen/Qwen3.5-2B-Base + supervised stages v1 to v8 + calibration-aware RL",
11
+ "layout": "plain",
12
+ "max_options": 255,
13
+ "max_state_tokens": 32768,
14
+ "schema_first": false,
15
+ "schema_first_trained": false,
16
+ "isolated_levels": true,
17
+ "release_date": "2026-09-24",
18
+ "requires": "decider-ai>=1.4.0 for temperature_by_type; older versions serve every answer at temperature",
19
+ "parent": "decider-2b v10 (Mapika/decider-2b, tag v10)",
20
+ "stage": "decider-2b v10 + LoRA rank 64 (alpha 128) on attention and MLP, LR 1e-4, 2 epochs (1,676 steps of 65,536 tokens) over 42,749 rows in the plain state-first layout (the same hard-decision rows as decider-4b v2.1 plus a broad replay of the public decision mixture), with the replay rows trained toward v10's own answer distribution (KL to v10) instead of their labels, merged into the bf16 weights; no new RL stage; temperature fitted by NLL on 61 in-task regression tasks; temperature_by_type fitted with decider.calibrate.fit_by_type on the same regression rows plus our own validation rows"
21
+ }
eval/hist.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "step": 303,
4
+ "agg": {
5
+ "in_task": {
6
+ "acc": 0.77,
7
+ "nll": 0.93548583984375,
8
+ "brier": 0.38499680161476135,
9
+ "ece": 0.18304700106382374,
10
+ "aurc": 0.08616249357070915,
11
+ "acc_at_80": 0.825,
12
+ "chance": 0.29995238095238097
13
+ },
14
+ "heldout": {},
15
+ "n_in": 1,
16
+ "n_heldout": 0
17
+ },
18
+ "results": {
19
+ "coreai_v2_heldout": {
20
+ "n": 100,
21
+ "acc": 0.77,
22
+ "nll": 0.93548583984375,
23
+ "brier": 0.38499680161476135,
24
+ "ece": 0.18304700106382374,
25
+ "aurc": 0.08616249357070915,
26
+ "acc_at_80": 0.825,
27
+ "acc_at_50": 0.94,
28
+ "chance": 0.29995238095238097,
29
+ "mean_conf": 0.943564772605896,
30
+ "sec": 25.9,
31
+ "heldout": false
32
+ }
33
+ }
34
+ }
35
+ ]
eval/jevbench_public231_baseline_decider-2b.json ADDED
@@ -0,0 +1,3646 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "Mapika/decider-2b",
3
+ "summary": {
4
+ "easy": {
5
+ "n": 48,
6
+ "accuracy": 1.0,
7
+ "invalid": 0,
8
+ "p50_s": 0.067
9
+ },
10
+ "standard": {
11
+ "n": 72,
12
+ "accuracy": 0.8889,
13
+ "invalid": 0,
14
+ "p50_s": 0.068
15
+ },
16
+ "hard": {
17
+ "n": 111,
18
+ "accuracy": 0.5586,
19
+ "invalid": 0,
20
+ "p50_s": 0.07
21
+ }
22
+ },
23
+ "rows": [
24
+ {
25
+ "id": "easy-intent-00",
26
+ "tier": "easy",
27
+ "expected": "track_order",
28
+ "predicted": "track_order",
29
+ "correct": true,
30
+ "valid": true,
31
+ "error": null,
32
+ "probs": {
33
+ "billing_question": 0.0001,
34
+ "cancel_order": 0.0003,
35
+ "change_address": 0.0001,
36
+ "report_damage": 0.0001,
37
+ "track_order": 0.9993
38
+ },
39
+ "latency_s": 48.479
40
+ },
41
+ {
42
+ "id": "easy-intent-01",
43
+ "tier": "easy",
44
+ "expected": "cancel_order",
45
+ "predicted": "cancel_order",
46
+ "correct": true,
47
+ "valid": true,
48
+ "error": null,
49
+ "probs": {
50
+ "billing_question": 0.0001,
51
+ "cancel_order": 0.9994,
52
+ "change_address": 0.0002,
53
+ "report_damage": 0.0001,
54
+ "track_order": 0.0001
55
+ },
56
+ "latency_s": 0.073
57
+ },
58
+ {
59
+ "id": "easy-intent-02",
60
+ "tier": "easy",
61
+ "expected": "change_address",
62
+ "predicted": "change_address",
63
+ "correct": true,
64
+ "valid": true,
65
+ "error": null,
66
+ "probs": {
67
+ "billing_question": 0.0001,
68
+ "cancel_order": 0.0003,
69
+ "change_address": 0.9991,
70
+ "report_damage": 0.0003,
71
+ "track_order": 0.0002
72
+ },
73
+ "latency_s": 0.067
74
+ },
75
+ {
76
+ "id": "easy-intent-03",
77
+ "tier": "easy",
78
+ "expected": "report_damage",
79
+ "predicted": "report_damage",
80
+ "correct": true,
81
+ "valid": true,
82
+ "error": null,
83
+ "probs": {
84
+ "billing_question": 0.0005,
85
+ "cancel_order": 0.0057,
86
+ "change_address": 0.0005,
87
+ "report_damage": 0.9926,
88
+ "track_order": 0.0007
89
+ },
90
+ "latency_s": 0.067
91
+ },
92
+ {
93
+ "id": "easy-intent-04",
94
+ "tier": "easy",
95
+ "expected": "billing_question",
96
+ "predicted": "billing_question",
97
+ "correct": true,
98
+ "valid": true,
99
+ "error": null,
100
+ "probs": {
101
+ "billing_question": 0.989,
102
+ "cancel_order": 0.006,
103
+ "change_address": 0.0018,
104
+ "report_damage": 0.0016,
105
+ "track_order": 0.0016
106
+ },
107
+ "latency_s": 0.067
108
+ },
109
+ {
110
+ "id": "easy-intent-05",
111
+ "tier": "easy",
112
+ "expected": "track_order",
113
+ "predicted": "track_order",
114
+ "correct": true,
115
+ "valid": true,
116
+ "error": null,
117
+ "probs": {
118
+ "billing_question": 0.0004,
119
+ "cancel_order": 0.0007,
120
+ "change_address": 0.0004,
121
+ "report_damage": 0.0002,
122
+ "track_order": 0.9983
123
+ },
124
+ "latency_s": 0.067
125
+ },
126
+ {
127
+ "id": "easy-intent-06",
128
+ "tier": "easy",
129
+ "expected": "set_alarm",
130
+ "predicted": "set_alarm",
131
+ "correct": true,
132
+ "valid": true,
133
+ "error": null,
134
+ "probs": {
135
+ "play_music": 0.0003,
136
+ "send_message": 0.0004,
137
+ "set_alarm": 0.9987,
138
+ "turn_off_lights": 0.0004,
139
+ "weather": 0.0002
140
+ },
141
+ "latency_s": 0.067
142
+ },
143
+ {
144
+ "id": "easy-intent-07",
145
+ "tier": "easy",
146
+ "expected": "play_music",
147
+ "predicted": "play_music",
148
+ "correct": true,
149
+ "valid": true,
150
+ "error": null,
151
+ "probs": {
152
+ "play_music": 0.9959,
153
+ "send_message": 0.0011,
154
+ "set_alarm": 0.0012,
155
+ "turn_off_lights": 0.001,
156
+ "weather": 0.0008
157
+ },
158
+ "latency_s": 0.067
159
+ },
160
+ {
161
+ "id": "easy-intent-08",
162
+ "tier": "easy",
163
+ "expected": "weather",
164
+ "predicted": "weather",
165
+ "correct": true,
166
+ "valid": true,
167
+ "error": null,
168
+ "probs": {
169
+ "play_music": 0.0001,
170
+ "send_message": 0.0001,
171
+ "set_alarm": 0.0001,
172
+ "turn_off_lights": 0.0,
173
+ "weather": 0.9996
174
+ },
175
+ "latency_s": 0.067
176
+ },
177
+ {
178
+ "id": "easy-intent-09",
179
+ "tier": "easy",
180
+ "expected": "send_message",
181
+ "predicted": "send_message",
182
+ "correct": true,
183
+ "valid": true,
184
+ "error": null,
185
+ "probs": {
186
+ "play_music": 0.0008,
187
+ "send_message": 0.9956,
188
+ "set_alarm": 0.0012,
189
+ "turn_off_lights": 0.0011,
190
+ "weather": 0.0013
191
+ },
192
+ "latency_s": 0.067
193
+ },
194
+ {
195
+ "id": "easy-intent-10",
196
+ "tier": "easy",
197
+ "expected": "turn_off_lights",
198
+ "predicted": "turn_off_lights",
199
+ "correct": true,
200
+ "valid": true,
201
+ "error": null,
202
+ "probs": {
203
+ "play_music": 0.0006,
204
+ "send_message": 0.0007,
205
+ "set_alarm": 0.001,
206
+ "turn_off_lights": 0.9972,
207
+ "weather": 0.0004
208
+ },
209
+ "latency_s": 0.067
210
+ },
211
+ {
212
+ "id": "easy-intent-11",
213
+ "tier": "easy",
214
+ "expected": "set_alarm",
215
+ "predicted": "set_alarm",
216
+ "correct": true,
217
+ "valid": true,
218
+ "error": null,
219
+ "probs": {
220
+ "play_music": 0.0002,
221
+ "send_message": 0.0002,
222
+ "set_alarm": 0.9991,
223
+ "turn_off_lights": 0.0003,
224
+ "weather": 0.0002
225
+ },
226
+ "latency_s": 0.067
227
+ },
228
+ {
229
+ "id": "easy-fact-00",
230
+ "tier": "easy",
231
+ "expected": "yes",
232
+ "predicted": "yes",
233
+ "correct": true,
234
+ "valid": true,
235
+ "error": null,
236
+ "probs": {
237
+ "yes": 0.9527,
238
+ "no": 0.0473
239
+ },
240
+ "latency_s": 0.068
241
+ },
242
+ {
243
+ "id": "easy-fact-01",
244
+ "tier": "easy",
245
+ "expected": "no",
246
+ "predicted": "no",
247
+ "correct": true,
248
+ "valid": true,
249
+ "error": null,
250
+ "probs": {
251
+ "yes": 0.0186,
252
+ "no": 0.9814
253
+ },
254
+ "latency_s": 0.068
255
+ },
256
+ {
257
+ "id": "easy-fact-02",
258
+ "tier": "easy",
259
+ "expected": "yes",
260
+ "predicted": "yes",
261
+ "correct": true,
262
+ "valid": true,
263
+ "error": null,
264
+ "probs": {
265
+ "yes": 0.9673,
266
+ "no": 0.0327
267
+ },
268
+ "latency_s": 0.067
269
+ },
270
+ {
271
+ "id": "easy-fact-03",
272
+ "tier": "easy",
273
+ "expected": "no",
274
+ "predicted": "no",
275
+ "correct": true,
276
+ "valid": true,
277
+ "error": null,
278
+ "probs": {
279
+ "yes": 0.0094,
280
+ "no": 0.9906
281
+ },
282
+ "latency_s": 0.07
283
+ },
284
+ {
285
+ "id": "easy-fact-04",
286
+ "tier": "easy",
287
+ "expected": "yes",
288
+ "predicted": "yes",
289
+ "correct": true,
290
+ "valid": true,
291
+ "error": null,
292
+ "probs": {
293
+ "yes": 0.9696,
294
+ "no": 0.0304
295
+ },
296
+ "latency_s": 0.067
297
+ },
298
+ {
299
+ "id": "easy-fact-05",
300
+ "tier": "easy",
301
+ "expected": "no",
302
+ "predicted": "no",
303
+ "correct": true,
304
+ "valid": true,
305
+ "error": null,
306
+ "probs": {
307
+ "yes": 0.0394,
308
+ "no": 0.9606
309
+ },
310
+ "latency_s": 0.067
311
+ },
312
+ {
313
+ "id": "easy-fact-06",
314
+ "tier": "easy",
315
+ "expected": "yes",
316
+ "predicted": "yes",
317
+ "correct": true,
318
+ "valid": true,
319
+ "error": null,
320
+ "probs": {
321
+ "yes": 0.9472,
322
+ "no": 0.0528
323
+ },
324
+ "latency_s": 0.067
325
+ },
326
+ {
327
+ "id": "easy-fact-07",
328
+ "tier": "easy",
329
+ "expected": "no",
330
+ "predicted": "no",
331
+ "correct": true,
332
+ "valid": true,
333
+ "error": null,
334
+ "probs": {
335
+ "yes": 0.0217,
336
+ "no": 0.9783
337
+ },
338
+ "latency_s": 0.067
339
+ },
340
+ {
341
+ "id": "easy-fact-08",
342
+ "tier": "easy",
343
+ "expected": "yes",
344
+ "predicted": "yes",
345
+ "correct": true,
346
+ "valid": true,
347
+ "error": null,
348
+ "probs": {
349
+ "yes": 0.9748,
350
+ "no": 0.0252
351
+ },
352
+ "latency_s": 0.077
353
+ },
354
+ {
355
+ "id": "easy-fact-09",
356
+ "tier": "easy",
357
+ "expected": "no",
358
+ "predicted": "no",
359
+ "correct": true,
360
+ "valid": true,
361
+ "error": null,
362
+ "probs": {
363
+ "yes": 0.0179,
364
+ "no": 0.9821
365
+ },
366
+ "latency_s": 0.068
367
+ },
368
+ {
369
+ "id": "easy-fact-10",
370
+ "tier": "easy",
371
+ "expected": "yes",
372
+ "predicted": "yes",
373
+ "correct": true,
374
+ "valid": true,
375
+ "error": null,
376
+ "probs": {
377
+ "yes": 0.9295,
378
+ "no": 0.0705
379
+ },
380
+ "latency_s": 0.067
381
+ },
382
+ {
383
+ "id": "easy-fact-11",
384
+ "tier": "easy",
385
+ "expected": "no",
386
+ "predicted": "no",
387
+ "correct": true,
388
+ "valid": true,
389
+ "error": null,
390
+ "probs": {
391
+ "yes": 0.0424,
392
+ "no": 0.9576
393
+ },
394
+ "latency_s": 0.067
395
+ },
396
+ {
397
+ "id": "easy-extraction-00",
398
+ "tier": "easy",
399
+ "expected": "paypal",
400
+ "predicted": "paypal",
401
+ "correct": true,
402
+ "valid": true,
403
+ "error": null,
404
+ "probs": {
405
+ "bank_transfer": 0.0005,
406
+ "cash": 0.0003,
407
+ "credit_card": 0.0015,
408
+ "paypal": 0.9976
409
+ },
410
+ "latency_s": 0.077
411
+ },
412
+ {
413
+ "id": "easy-extraction-01",
414
+ "tier": "easy",
415
+ "expected": "cash",
416
+ "predicted": "cash",
417
+ "correct": true,
418
+ "valid": true,
419
+ "error": null,
420
+ "probs": {
421
+ "bank_transfer": 0.0008,
422
+ "cash": 0.9933,
423
+ "credit_card": 0.0052,
424
+ "paypal": 0.0007
425
+ },
426
+ "latency_s": 0.066
427
+ },
428
+ {
429
+ "id": "easy-extraction-02",
430
+ "tier": "easy",
431
+ "expected": "bank_transfer",
432
+ "predicted": "bank_transfer",
433
+ "correct": true,
434
+ "valid": true,
435
+ "error": null,
436
+ "probs": {
437
+ "bank_transfer": 0.9969,
438
+ "cash": 0.0007,
439
+ "credit_card": 0.0014,
440
+ "paypal": 0.001
441
+ },
442
+ "latency_s": 0.066
443
+ },
444
+ {
445
+ "id": "easy-extraction-03",
446
+ "tier": "easy",
447
+ "expected": "high",
448
+ "predicted": "high",
449
+ "correct": true,
450
+ "valid": true,
451
+ "error": null,
452
+ "probs": {
453
+ "high": 0.9102,
454
+ "low": 0.0062,
455
+ "medium": 0.0181,
456
+ "urgent": 0.0655
457
+ },
458
+ "latency_s": 0.067
459
+ },
460
+ {
461
+ "id": "easy-extraction-04",
462
+ "tier": "easy",
463
+ "expected": "low",
464
+ "predicted": "low",
465
+ "correct": true,
466
+ "valid": true,
467
+ "error": null,
468
+ "probs": {
469
+ "high": 0.0057,
470
+ "low": 0.9794,
471
+ "medium": 0.0133,
472
+ "urgent": 0.0016
473
+ },
474
+ "latency_s": 0.066
475
+ },
476
+ {
477
+ "id": "easy-extraction-05",
478
+ "tier": "easy",
479
+ "expected": "urgent",
480
+ "predicted": "urgent",
481
+ "correct": true,
482
+ "valid": true,
483
+ "error": null,
484
+ "probs": {
485
+ "high": 0.0114,
486
+ "low": 0.0014,
487
+ "medium": 0.0033,
488
+ "urgent": 0.9839
489
+ },
490
+ "latency_s": 0.066
491
+ },
492
+ {
493
+ "id": "easy-extraction-06",
494
+ "tier": "easy",
495
+ "expected": "M",
496
+ "predicted": "M",
497
+ "correct": true,
498
+ "valid": true,
499
+ "error": null,
500
+ "probs": {
501
+ "L": 0.0051,
502
+ "M": 0.9912,
503
+ "S": 0.0021,
504
+ "XL": 0.0016
505
+ },
506
+ "latency_s": 0.066
507
+ },
508
+ {
509
+ "id": "easy-extraction-07",
510
+ "tier": "easy",
511
+ "expected": "XL",
512
+ "predicted": "XL",
513
+ "correct": true,
514
+ "valid": true,
515
+ "error": null,
516
+ "probs": {
517
+ "L": 0.0049,
518
+ "M": 0.001,
519
+ "S": 0.0008,
520
+ "XL": 0.9933
521
+ },
522
+ "latency_s": 0.066
523
+ },
524
+ {
525
+ "id": "easy-extraction-08",
526
+ "tier": "easy",
527
+ "expected": "thursday",
528
+ "predicted": "thursday",
529
+ "correct": true,
530
+ "valid": true,
531
+ "error": null,
532
+ "probs": {
533
+ "friday": 0.0041,
534
+ "monday": 0.0024,
535
+ "thursday": 0.9904,
536
+ "tuesday": 0.0017,
537
+ "wednesday": 0.0013
538
+ },
539
+ "latency_s": 0.067
540
+ },
541
+ {
542
+ "id": "easy-extraction-09",
543
+ "tier": "easy",
544
+ "expected": "monday",
545
+ "predicted": "monday",
546
+ "correct": true,
547
+ "valid": true,
548
+ "error": null,
549
+ "probs": {
550
+ "friday": 0.0289,
551
+ "monday": 0.8996,
552
+ "thursday": 0.0246,
553
+ "tuesday": 0.0289,
554
+ "wednesday": 0.0179
555
+ },
556
+ "latency_s": 0.066
557
+ },
558
+ {
559
+ "id": "easy-extraction-10",
560
+ "tier": "easy",
561
+ "expected": "friday",
562
+ "predicted": "friday",
563
+ "correct": true,
564
+ "valid": true,
565
+ "error": null,
566
+ "probs": {
567
+ "friday": 0.9774,
568
+ "monday": 0.0048,
569
+ "thursday": 0.0074,
570
+ "tuesday": 0.0053,
571
+ "wednesday": 0.0051
572
+ },
573
+ "latency_s": 0.064
574
+ },
575
+ {
576
+ "id": "easy-extraction-11",
577
+ "tier": "easy",
578
+ "expected": "credit_card",
579
+ "predicted": "credit_card",
580
+ "correct": true,
581
+ "valid": true,
582
+ "error": null,
583
+ "probs": {
584
+ "bank_transfer": 0.0007,
585
+ "cash": 0.0004,
586
+ "credit_card": 0.9978,
587
+ "paypal": 0.0011
588
+ },
589
+ "latency_s": 0.066
590
+ },
591
+ {
592
+ "id": "easy-tool_selection-00",
593
+ "tier": "easy",
594
+ "expected": "get_weather",
595
+ "predicted": "get_weather",
596
+ "correct": true,
597
+ "valid": true,
598
+ "error": null,
599
+ "probs": {
600
+ "convert_currency": 0.0001,
601
+ "create_calendar_event": 0.0002,
602
+ "get_weather": 0.9992,
603
+ "send_email": 0.0003,
604
+ "translate_text": 0.0003
605
+ },
606
+ "latency_s": 0.066
607
+ },
608
+ {
609
+ "id": "easy-tool_selection-01",
610
+ "tier": "easy",
611
+ "expected": "send_email",
612
+ "predicted": "send_email",
613
+ "correct": true,
614
+ "valid": true,
615
+ "error": null,
616
+ "probs": {
617
+ "convert_currency": 0.0003,
618
+ "create_calendar_event": 0.0004,
619
+ "get_weather": 0.0004,
620
+ "send_email": 0.9983,
621
+ "translate_text": 0.0006
622
+ },
623
+ "latency_s": 0.066
624
+ },
625
+ {
626
+ "id": "easy-tool_selection-02",
627
+ "tier": "easy",
628
+ "expected": "create_calendar_event",
629
+ "predicted": "create_calendar_event",
630
+ "correct": true,
631
+ "valid": true,
632
+ "error": null,
633
+ "probs": {
634
+ "convert_currency": 0.0001,
635
+ "create_calendar_event": 0.9988,
636
+ "get_weather": 0.0004,
637
+ "send_email": 0.0003,
638
+ "translate_text": 0.0004
639
+ },
640
+ "latency_s": 0.066
641
+ },
642
+ {
643
+ "id": "easy-tool_selection-03",
644
+ "tier": "easy",
645
+ "expected": "convert_currency",
646
+ "predicted": "convert_currency",
647
+ "correct": true,
648
+ "valid": true,
649
+ "error": null,
650
+ "probs": {
651
+ "convert_currency": 0.998,
652
+ "create_calendar_event": 0.0004,
653
+ "get_weather": 0.0005,
654
+ "send_email": 0.0004,
655
+ "translate_text": 0.0007
656
+ },
657
+ "latency_s": 0.066
658
+ },
659
+ {
660
+ "id": "easy-tool_selection-04",
661
+ "tier": "easy",
662
+ "expected": "translate_text",
663
+ "predicted": "translate_text",
664
+ "correct": true,
665
+ "valid": true,
666
+ "error": null,
667
+ "probs": {
668
+ "convert_currency": 0.0002,
669
+ "create_calendar_event": 0.0002,
670
+ "get_weather": 0.0003,
671
+ "send_email": 0.0001,
672
+ "translate_text": 0.9992
673
+ },
674
+ "latency_s": 0.066
675
+ },
676
+ {
677
+ "id": "easy-tool_selection-05",
678
+ "tier": "easy",
679
+ "expected": "create_calendar_event",
680
+ "predicted": "create_calendar_event",
681
+ "correct": true,
682
+ "valid": true,
683
+ "error": null,
684
+ "probs": {
685
+ "convert_currency": 0.0002,
686
+ "create_calendar_event": 0.998,
687
+ "get_weather": 0.0006,
688
+ "send_email": 0.0007,
689
+ "translate_text": 0.0004
690
+ },
691
+ "latency_s": 0.067
692
+ },
693
+ {
694
+ "id": "easy-tool_selection-06",
695
+ "tier": "easy",
696
+ "expected": "search_flights",
697
+ "predicted": "search_flights",
698
+ "correct": true,
699
+ "valid": true,
700
+ "error": null,
701
+ "probs": {
702
+ "book_restaurant": 0.0001,
703
+ "call_taxi": 0.0001,
704
+ "get_stock_price": 0.0002,
705
+ "search_flights": 0.9993,
706
+ "set_reminder": 0.0003
707
+ },
708
+ "latency_s": 0.066
709
+ },
710
+ {
711
+ "id": "easy-tool_selection-07",
712
+ "tier": "easy",
713
+ "expected": "book_restaurant",
714
+ "predicted": "book_restaurant",
715
+ "correct": true,
716
+ "valid": true,
717
+ "error": null,
718
+ "probs": {
719
+ "book_restaurant": 0.9951,
720
+ "call_taxi": 0.0005,
721
+ "get_stock_price": 0.0008,
722
+ "search_flights": 0.0006,
723
+ "set_reminder": 0.003
724
+ },
725
+ "latency_s": 0.066
726
+ },
727
+ {
728
+ "id": "easy-tool_selection-08",
729
+ "tier": "easy",
730
+ "expected": "set_reminder",
731
+ "predicted": "set_reminder",
732
+ "correct": true,
733
+ "valid": true,
734
+ "error": null,
735
+ "probs": {
736
+ "book_restaurant": 0.0002,
737
+ "call_taxi": 0.0003,
738
+ "get_stock_price": 0.0004,
739
+ "search_flights": 0.0002,
740
+ "set_reminder": 0.9989
741
+ },
742
+ "latency_s": 0.066
743
+ },
744
+ {
745
+ "id": "easy-tool_selection-09",
746
+ "tier": "easy",
747
+ "expected": "get_stock_price",
748
+ "predicted": "get_stock_price",
749
+ "correct": true,
750
+ "valid": true,
751
+ "error": null,
752
+ "probs": {
753
+ "book_restaurant": 0.0001,
754
+ "call_taxi": 0.0002,
755
+ "get_stock_price": 0.9993,
756
+ "search_flights": 0.0002,
757
+ "set_reminder": 0.0002
758
+ },
759
+ "latency_s": 0.066
760
+ },
761
+ {
762
+ "id": "easy-tool_selection-10",
763
+ "tier": "easy",
764
+ "expected": "call_taxi",
765
+ "predicted": "call_taxi",
766
+ "correct": true,
767
+ "valid": true,
768
+ "error": null,
769
+ "probs": {
770
+ "book_restaurant": 0.0005,
771
+ "call_taxi": 0.9966,
772
+ "get_stock_price": 0.0009,
773
+ "search_flights": 0.0007,
774
+ "set_reminder": 0.0013
775
+ },
776
+ "latency_s": 0.066
777
+ },
778
+ {
779
+ "id": "easy-tool_selection-11",
780
+ "tier": "easy",
781
+ "expected": "set_reminder",
782
+ "predicted": "set_reminder",
783
+ "correct": true,
784
+ "valid": true,
785
+ "error": null,
786
+ "probs": {
787
+ "book_restaurant": 0.0001,
788
+ "call_taxi": 0.0002,
789
+ "get_stock_price": 0.0002,
790
+ "search_flights": 0.0001,
791
+ "set_reminder": 0.9994
792
+ },
793
+ "latency_s": 0.067
794
+ },
795
+ {
796
+ "id": "original-policy-01-0",
797
+ "tier": "standard",
798
+ "expected": "no",
799
+ "predicted": "no",
800
+ "correct": true,
801
+ "valid": true,
802
+ "error": null,
803
+ "probs": {
804
+ "yes": 0.0473,
805
+ "no": 0.9527
806
+ },
807
+ "latency_s": 0.066
808
+ },
809
+ {
810
+ "id": "original-policy-01-1",
811
+ "tier": "standard",
812
+ "expected": "no",
813
+ "predicted": "no",
814
+ "correct": true,
815
+ "valid": true,
816
+ "error": null,
817
+ "probs": {
818
+ "yes": 0.0473,
819
+ "no": 0.9527
820
+ },
821
+ "latency_s": 0.066
822
+ },
823
+ {
824
+ "id": "original-policy-02-0",
825
+ "tier": "standard",
826
+ "expected": "yes",
827
+ "predicted": "yes",
828
+ "correct": true,
829
+ "valid": true,
830
+ "error": null,
831
+ "probs": {
832
+ "yes": 0.8683,
833
+ "no": 0.1317
834
+ },
835
+ "latency_s": 0.066
836
+ },
837
+ {
838
+ "id": "original-policy-02-1",
839
+ "tier": "standard",
840
+ "expected": "yes",
841
+ "predicted": "no",
842
+ "correct": false,
843
+ "valid": true,
844
+ "error": null,
845
+ "probs": {
846
+ "yes": 0.2921,
847
+ "no": 0.7079
848
+ },
849
+ "latency_s": 0.066
850
+ },
851
+ {
852
+ "id": "original-policy-03-0",
853
+ "tier": "standard",
854
+ "expected": "yes",
855
+ "predicted": "yes",
856
+ "correct": true,
857
+ "valid": true,
858
+ "error": null,
859
+ "probs": {
860
+ "yes": 0.5288,
861
+ "no": 0.4712
862
+ },
863
+ "latency_s": 0.067
864
+ },
865
+ {
866
+ "id": "original-policy-03-1",
867
+ "tier": "standard",
868
+ "expected": "yes",
869
+ "predicted": "no",
870
+ "correct": false,
871
+ "valid": true,
872
+ "error": null,
873
+ "probs": {
874
+ "yes": 0.405,
875
+ "no": 0.595
876
+ },
877
+ "latency_s": 0.066
878
+ },
879
+ {
880
+ "id": "original-policy-04-0",
881
+ "tier": "standard",
882
+ "expected": "no",
883
+ "predicted": "no",
884
+ "correct": true,
885
+ "valid": true,
886
+ "error": null,
887
+ "probs": {
888
+ "yes": 0.1605,
889
+ "no": 0.8395
890
+ },
891
+ "latency_s": 0.066
892
+ },
893
+ {
894
+ "id": "original-policy-04-1",
895
+ "tier": "standard",
896
+ "expected": "no",
897
+ "predicted": "no",
898
+ "correct": true,
899
+ "valid": true,
900
+ "error": null,
901
+ "probs": {
902
+ "yes": 0.0785,
903
+ "no": 0.9215
904
+ },
905
+ "latency_s": 0.066
906
+ },
907
+ {
908
+ "id": "original-policy-05-0",
909
+ "tier": "standard",
910
+ "expected": "no",
911
+ "predicted": "no",
912
+ "correct": true,
913
+ "valid": true,
914
+ "error": null,
915
+ "probs": {
916
+ "yes": 0.3249,
917
+ "no": 0.6751
918
+ },
919
+ "latency_s": 0.066
920
+ },
921
+ {
922
+ "id": "original-policy-05-1",
923
+ "tier": "standard",
924
+ "expected": "no",
925
+ "predicted": "no",
926
+ "correct": true,
927
+ "valid": true,
928
+ "error": null,
929
+ "probs": {
930
+ "yes": 0.1553,
931
+ "no": 0.8447
932
+ },
933
+ "latency_s": 0.066
934
+ },
935
+ {
936
+ "id": "original-policy-06-0",
937
+ "tier": "standard",
938
+ "expected": "yes",
939
+ "predicted": "yes",
940
+ "correct": true,
941
+ "valid": true,
942
+ "error": null,
943
+ "probs": {
944
+ "yes": 0.8289,
945
+ "no": 0.1711
946
+ },
947
+ "latency_s": 0.066
948
+ },
949
+ {
950
+ "id": "original-policy-06-1",
951
+ "tier": "standard",
952
+ "expected": "yes",
953
+ "predicted": "yes",
954
+ "correct": true,
955
+ "valid": true,
956
+ "error": null,
957
+ "probs": {
958
+ "yes": 0.7235,
959
+ "no": 0.2765
960
+ },
961
+ "latency_s": 0.066
962
+ },
963
+ {
964
+ "id": "original-intent-01-0",
965
+ "tier": "standard",
966
+ "expected": "refund",
967
+ "predicted": "refund",
968
+ "correct": true,
969
+ "valid": true,
970
+ "error": null,
971
+ "probs": {
972
+ "cancel": 0.0256,
973
+ "change_address": 0.0016,
974
+ "other": 0.0373,
975
+ "refund": 0.9349,
976
+ "status": 0.0006
977
+ },
978
+ "latency_s": 0.066
979
+ },
980
+ {
981
+ "id": "original-intent-01-1",
982
+ "tier": "standard",
983
+ "expected": "refund",
984
+ "predicted": "refund",
985
+ "correct": true,
986
+ "valid": true,
987
+ "error": null,
988
+ "probs": {
989
+ "cancel": 0.0896,
990
+ "change_address": 0.0027,
991
+ "other": 0.0524,
992
+ "refund": 0.8543,
993
+ "status": 0.001
994
+ },
995
+ "latency_s": 0.067
996
+ },
997
+ {
998
+ "id": "original-intent-02-0",
999
+ "tier": "standard",
1000
+ "expected": "status",
1001
+ "predicted": "status",
1002
+ "correct": true,
1003
+ "valid": true,
1004
+ "error": null,
1005
+ "probs": {
1006
+ "cancel": 0.0015,
1007
+ "change_address": 0.009,
1008
+ "other": 0.2514,
1009
+ "refund": 0.0025,
1010
+ "status": 0.7357
1011
+ },
1012
+ "latency_s": 0.067
1013
+ },
1014
+ {
1015
+ "id": "original-intent-02-1",
1016
+ "tier": "standard",
1017
+ "expected": "status",
1018
+ "predicted": "status",
1019
+ "correct": true,
1020
+ "valid": true,
1021
+ "error": null,
1022
+ "probs": {
1023
+ "cancel": 0.0003,
1024
+ "change_address": 0.001,
1025
+ "other": 0.0194,
1026
+ "refund": 0.0003,
1027
+ "status": 0.9789
1028
+ },
1029
+ "latency_s": 0.066
1030
+ },
1031
+ {
1032
+ "id": "original-intent-03-0",
1033
+ "tier": "standard",
1034
+ "expected": "change_address",
1035
+ "predicted": "change_address",
1036
+ "correct": true,
1037
+ "valid": true,
1038
+ "error": null,
1039
+ "probs": {
1040
+ "cancel": 0.0028,
1041
+ "change_address": 0.8751,
1042
+ "other": 0.1078,
1043
+ "refund": 0.0074,
1044
+ "status": 0.007
1045
+ },
1046
+ "latency_s": 0.066
1047
+ },
1048
+ {
1049
+ "id": "original-intent-03-1",
1050
+ "tier": "standard",
1051
+ "expected": "change_address",
1052
+ "predicted": "change_address",
1053
+ "correct": true,
1054
+ "valid": true,
1055
+ "error": null,
1056
+ "probs": {
1057
+ "cancel": 0.0003,
1058
+ "change_address": 0.986,
1059
+ "other": 0.0127,
1060
+ "refund": 0.0007,
1061
+ "status": 0.0003
1062
+ },
1063
+ "latency_s": 0.066
1064
+ },
1065
+ {
1066
+ "id": "original-intent-04-0",
1067
+ "tier": "standard",
1068
+ "expected": "cancel",
1069
+ "predicted": "cancel",
1070
+ "correct": true,
1071
+ "valid": true,
1072
+ "error": null,
1073
+ "probs": {
1074
+ "cancel": 0.9461,
1075
+ "change_address": 0.0019,
1076
+ "other": 0.0377,
1077
+ "refund": 0.0136,
1078
+ "status": 0.0007
1079
+ },
1080
+ "latency_s": 0.065
1081
+ },
1082
+ {
1083
+ "id": "original-intent-04-1",
1084
+ "tier": "standard",
1085
+ "expected": "cancel",
1086
+ "predicted": "cancel",
1087
+ "correct": true,
1088
+ "valid": true,
1089
+ "error": null,
1090
+ "probs": {
1091
+ "cancel": 0.9607,
1092
+ "change_address": 0.0013,
1093
+ "other": 0.0326,
1094
+ "refund": 0.0047,
1095
+ "status": 0.0006
1096
+ },
1097
+ "latency_s": 0.066
1098
+ },
1099
+ {
1100
+ "id": "original-intent-05-0",
1101
+ "tier": "standard",
1102
+ "expected": "other",
1103
+ "predicted": "other",
1104
+ "correct": true,
1105
+ "valid": true,
1106
+ "error": null,
1107
+ "probs": {
1108
+ "cancel": 0.0029,
1109
+ "change_address": 0.0025,
1110
+ "other": 0.7783,
1111
+ "refund": 0.2145,
1112
+ "status": 0.0017
1113
+ },
1114
+ "latency_s": 0.066
1115
+ },
1116
+ {
1117
+ "id": "original-intent-05-1",
1118
+ "tier": "standard",
1119
+ "expected": "other",
1120
+ "predicted": "other",
1121
+ "correct": true,
1122
+ "valid": true,
1123
+ "error": null,
1124
+ "probs": {
1125
+ "cancel": 0.0105,
1126
+ "change_address": 0.0032,
1127
+ "other": 0.6212,
1128
+ "refund": 0.3631,
1129
+ "status": 0.0019
1130
+ },
1131
+ "latency_s": 0.065
1132
+ },
1133
+ {
1134
+ "id": "original-intent-06-0",
1135
+ "tier": "standard",
1136
+ "expected": "status",
1137
+ "predicted": "status",
1138
+ "correct": true,
1139
+ "valid": true,
1140
+ "error": null,
1141
+ "probs": {
1142
+ "cancel": 0.0016,
1143
+ "change_address": 0.0006,
1144
+ "other": 0.0266,
1145
+ "refund": 0.0009,
1146
+ "status": 0.9703
1147
+ },
1148
+ "latency_s": 0.065
1149
+ },
1150
+ {
1151
+ "id": "original-intent-06-1",
1152
+ "tier": "standard",
1153
+ "expected": "status",
1154
+ "predicted": "status",
1155
+ "correct": true,
1156
+ "valid": true,
1157
+ "error": null,
1158
+ "probs": {
1159
+ "cancel": 0.0007,
1160
+ "change_address": 0.0003,
1161
+ "other": 0.0166,
1162
+ "refund": 0.0007,
1163
+ "status": 0.9816
1164
+ },
1165
+ "latency_s": 0.066
1166
+ },
1167
+ {
1168
+ "id": "original-ordinal-01-0",
1169
+ "tier": "standard",
1170
+ "expected": "0",
1171
+ "predicted": "0",
1172
+ "correct": true,
1173
+ "valid": true,
1174
+ "error": null,
1175
+ "probs": {
1176
+ "0": 0.8232,
1177
+ "1": 0.1672,
1178
+ "2": 0.0062,
1179
+ "3": 0.0034
1180
+ },
1181
+ "latency_s": 0.805
1182
+ },
1183
+ {
1184
+ "id": "original-ordinal-01-1",
1185
+ "tier": "standard",
1186
+ "expected": "0",
1187
+ "predicted": "0",
1188
+ "correct": true,
1189
+ "valid": true,
1190
+ "error": null,
1191
+ "probs": {
1192
+ "0": 0.7581,
1193
+ "1": 0.2277,
1194
+ "2": 0.0111,
1195
+ "3": 0.0031
1196
+ },
1197
+ "latency_s": 0.068
1198
+ },
1199
+ {
1200
+ "id": "original-ordinal-02-0",
1201
+ "tier": "standard",
1202
+ "expected": "1",
1203
+ "predicted": "1",
1204
+ "correct": true,
1205
+ "valid": true,
1206
+ "error": null,
1207
+ "probs": {
1208
+ "0": 0.0748,
1209
+ "1": 0.7906,
1210
+ "2": 0.1256,
1211
+ "3": 0.009
1212
+ },
1213
+ "latency_s": 0.068
1214
+ },
1215
+ {
1216
+ "id": "original-ordinal-02-1",
1217
+ "tier": "standard",
1218
+ "expected": "1",
1219
+ "predicted": "1",
1220
+ "correct": true,
1221
+ "valid": true,
1222
+ "error": null,
1223
+ "probs": {
1224
+ "0": 0.0672,
1225
+ "1": 0.9059,
1226
+ "2": 0.0174,
1227
+ "3": 0.0095
1228
+ },
1229
+ "latency_s": 0.068
1230
+ },
1231
+ {
1232
+ "id": "original-ordinal-03-0",
1233
+ "tier": "standard",
1234
+ "expected": "2",
1235
+ "predicted": "2",
1236
+ "correct": true,
1237
+ "valid": true,
1238
+ "error": null,
1239
+ "probs": {
1240
+ "0": 0.0872,
1241
+ "1": 0.1283,
1242
+ "2": 0.7794,
1243
+ "3": 0.0051
1244
+ },
1245
+ "latency_s": 0.068
1246
+ },
1247
+ {
1248
+ "id": "original-ordinal-03-1",
1249
+ "tier": "standard",
1250
+ "expected": "2",
1251
+ "predicted": "2",
1252
+ "correct": true,
1253
+ "valid": true,
1254
+ "error": null,
1255
+ "probs": {
1256
+ "0": 0.064,
1257
+ "1": 0.1854,
1258
+ "2": 0.7347,
1259
+ "3": 0.0159
1260
+ },
1261
+ "latency_s": 0.068
1262
+ },
1263
+ {
1264
+ "id": "original-ordinal-04-0",
1265
+ "tier": "standard",
1266
+ "expected": "3",
1267
+ "predicted": "3",
1268
+ "correct": true,
1269
+ "valid": true,
1270
+ "error": null,
1271
+ "probs": {
1272
+ "0": 0.0162,
1273
+ "1": 0.0162,
1274
+ "2": 0.0452,
1275
+ "3": 0.9225
1276
+ },
1277
+ "latency_s": 0.069
1278
+ },
1279
+ {
1280
+ "id": "original-ordinal-04-1",
1281
+ "tier": "standard",
1282
+ "expected": "3",
1283
+ "predicted": "3",
1284
+ "correct": true,
1285
+ "valid": true,
1286
+ "error": null,
1287
+ "probs": {
1288
+ "0": 0.024,
1289
+ "1": 0.0268,
1290
+ "2": 0.129,
1291
+ "3": 0.8201
1292
+ },
1293
+ "latency_s": 0.069
1294
+ },
1295
+ {
1296
+ "id": "original-ordinal-05-0",
1297
+ "tier": "standard",
1298
+ "expected": "0",
1299
+ "predicted": "0",
1300
+ "correct": true,
1301
+ "valid": true,
1302
+ "error": null,
1303
+ "probs": {
1304
+ "0": 0.8768,
1305
+ "1": 0.0851,
1306
+ "2": 0.0323,
1307
+ "3": 0.0059
1308
+ },
1309
+ "latency_s": 0.07
1310
+ },
1311
+ {
1312
+ "id": "original-ordinal-05-1",
1313
+ "tier": "standard",
1314
+ "expected": "0",
1315
+ "predicted": "0",
1316
+ "correct": true,
1317
+ "valid": true,
1318
+ "error": null,
1319
+ "probs": {
1320
+ "0": 0.7539,
1321
+ "1": 0.0969,
1322
+ "2": 0.1273,
1323
+ "3": 0.0219
1324
+ },
1325
+ "latency_s": 0.07
1326
+ },
1327
+ {
1328
+ "id": "original-ordinal-06-0",
1329
+ "tier": "standard",
1330
+ "expected": "2",
1331
+ "predicted": "2",
1332
+ "correct": true,
1333
+ "valid": true,
1334
+ "error": null,
1335
+ "probs": {
1336
+ "0": 0.087,
1337
+ "1": 0.1538,
1338
+ "2": 0.757,
1339
+ "3": 0.0022
1340
+ },
1341
+ "latency_s": 0.069
1342
+ },
1343
+ {
1344
+ "id": "original-ordinal-06-1",
1345
+ "tier": "standard",
1346
+ "expected": "2",
1347
+ "predicted": "2",
1348
+ "correct": true,
1349
+ "valid": true,
1350
+ "error": null,
1351
+ "probs": {
1352
+ "0": 0.0293,
1353
+ "1": 0.1684,
1354
+ "2": 0.7996,
1355
+ "3": 0.0028
1356
+ },
1357
+ "latency_s": 0.069
1358
+ },
1359
+ {
1360
+ "id": "original-extraction-01-0",
1361
+ "tier": "standard",
1362
+ "expected": "pickup",
1363
+ "predicted": "pickup",
1364
+ "correct": true,
1365
+ "valid": true,
1366
+ "error": null,
1367
+ "probs": {
1368
+ "courier": 0.0143,
1369
+ "pickup": 0.8908,
1370
+ "post": 0.0064,
1371
+ "unknown": 0.0885
1372
+ },
1373
+ "latency_s": 0.068
1374
+ },
1375
+ {
1376
+ "id": "original-extraction-01-1",
1377
+ "tier": "standard",
1378
+ "expected": "pickup",
1379
+ "predicted": "pickup",
1380
+ "correct": true,
1381
+ "valid": true,
1382
+ "error": null,
1383
+ "probs": {
1384
+ "courier": 0.0045,
1385
+ "pickup": 0.9083,
1386
+ "post": 0.0062,
1387
+ "unknown": 0.0811
1388
+ },
1389
+ "latency_s": 0.069
1390
+ },
1391
+ {
1392
+ "id": "original-extraction-02-0",
1393
+ "tier": "standard",
1394
+ "expected": "unknown",
1395
+ "predicted": "unknown",
1396
+ "correct": true,
1397
+ "valid": true,
1398
+ "error": null,
1399
+ "probs": {
1400
+ "courier": 0.0274,
1401
+ "pickup": 0.008,
1402
+ "post": 0.1998,
1403
+ "unknown": 0.7648
1404
+ },
1405
+ "latency_s": 0.072
1406
+ },
1407
+ {
1408
+ "id": "original-extraction-02-1",
1409
+ "tier": "standard",
1410
+ "expected": "unknown",
1411
+ "predicted": "unknown",
1412
+ "correct": true,
1413
+ "valid": true,
1414
+ "error": null,
1415
+ "probs": {
1416
+ "courier": 0.0095,
1417
+ "pickup": 0.0034,
1418
+ "post": 0.0733,
1419
+ "unknown": 0.9138
1420
+ },
1421
+ "latency_s": 0.069
1422
+ },
1423
+ {
1424
+ "id": "original-extraction-03-0",
1425
+ "tier": "standard",
1426
+ "expected": "courier",
1427
+ "predicted": "courier",
1428
+ "correct": true,
1429
+ "valid": true,
1430
+ "error": null,
1431
+ "probs": {
1432
+ "courier": 0.6255,
1433
+ "pickup": 0.045,
1434
+ "post": 0.0344,
1435
+ "unknown": 0.295
1436
+ },
1437
+ "latency_s": 0.069
1438
+ },
1439
+ {
1440
+ "id": "original-extraction-03-1",
1441
+ "tier": "standard",
1442
+ "expected": "courier",
1443
+ "predicted": "courier",
1444
+ "correct": true,
1445
+ "valid": true,
1446
+ "error": null,
1447
+ "probs": {
1448
+ "courier": 0.985,
1449
+ "pickup": 0.0014,
1450
+ "post": 0.0048,
1451
+ "unknown": 0.0087
1452
+ },
1453
+ "latency_s": 0.068
1454
+ },
1455
+ {
1456
+ "id": "original-extraction-04-0",
1457
+ "tier": "standard",
1458
+ "expected": "post",
1459
+ "predicted": "post",
1460
+ "correct": true,
1461
+ "valid": true,
1462
+ "error": null,
1463
+ "probs": {
1464
+ "courier": 0.002,
1465
+ "pickup": 0.0018,
1466
+ "post": 0.9757,
1467
+ "unknown": 0.0204
1468
+ },
1469
+ "latency_s": 0.067
1470
+ },
1471
+ {
1472
+ "id": "original-extraction-04-1",
1473
+ "tier": "standard",
1474
+ "expected": "post",
1475
+ "predicted": "post",
1476
+ "correct": true,
1477
+ "valid": true,
1478
+ "error": null,
1479
+ "probs": {
1480
+ "courier": 0.0065,
1481
+ "pickup": 0.0027,
1482
+ "post": 0.9547,
1483
+ "unknown": 0.0361
1484
+ },
1485
+ "latency_s": 0.068
1486
+ },
1487
+ {
1488
+ "id": "original-extraction-05-0",
1489
+ "tier": "standard",
1490
+ "expected": "unknown",
1491
+ "predicted": "unknown",
1492
+ "correct": true,
1493
+ "valid": true,
1494
+ "error": null,
1495
+ "probs": {
1496
+ "courier": 0.0054,
1497
+ "pickup": 0.006,
1498
+ "post": 0.0037,
1499
+ "unknown": 0.9849
1500
+ },
1501
+ "latency_s": 0.068
1502
+ },
1503
+ {
1504
+ "id": "original-extraction-05-1",
1505
+ "tier": "standard",
1506
+ "expected": "unknown",
1507
+ "predicted": "unknown",
1508
+ "correct": true,
1509
+ "valid": true,
1510
+ "error": null,
1511
+ "probs": {
1512
+ "courier": 0.3457,
1513
+ "pickup": 0.0111,
1514
+ "post": 0.019,
1515
+ "unknown": 0.6241
1516
+ },
1517
+ "latency_s": 0.067
1518
+ },
1519
+ {
1520
+ "id": "original-extraction-06-0",
1521
+ "tier": "standard",
1522
+ "expected": "courier",
1523
+ "predicted": "courier",
1524
+ "correct": true,
1525
+ "valid": true,
1526
+ "error": null,
1527
+ "probs": {
1528
+ "courier": 0.9661,
1529
+ "pickup": 0.0033,
1530
+ "post": 0.0081,
1531
+ "unknown": 0.0225
1532
+ },
1533
+ "latency_s": 0.068
1534
+ },
1535
+ {
1536
+ "id": "original-extraction-06-1",
1537
+ "tier": "standard",
1538
+ "expected": "courier",
1539
+ "predicted": "courier",
1540
+ "correct": true,
1541
+ "valid": true,
1542
+ "error": null,
1543
+ "probs": {
1544
+ "courier": 0.9735,
1545
+ "pickup": 0.0024,
1546
+ "post": 0.0048,
1547
+ "unknown": 0.0193
1548
+ },
1549
+ "latency_s": 0.068
1550
+ },
1551
+ {
1552
+ "id": "original-adequacy-01-0",
1553
+ "tier": "standard",
1554
+ "expected": "yes",
1555
+ "predicted": "yes",
1556
+ "correct": true,
1557
+ "valid": true,
1558
+ "error": null,
1559
+ "probs": {
1560
+ "yes": 0.8496,
1561
+ "no": 0.1504
1562
+ },
1563
+ "latency_s": 0.068
1564
+ },
1565
+ {
1566
+ "id": "original-adequacy-01-1",
1567
+ "tier": "standard",
1568
+ "expected": "yes",
1569
+ "predicted": "yes",
1570
+ "correct": true,
1571
+ "valid": true,
1572
+ "error": null,
1573
+ "probs": {
1574
+ "yes": 0.7533,
1575
+ "no": 0.2467
1576
+ },
1577
+ "latency_s": 0.068
1578
+ },
1579
+ {
1580
+ "id": "original-adequacy-02-0",
1581
+ "tier": "standard",
1582
+ "expected": "no",
1583
+ "predicted": "no",
1584
+ "correct": true,
1585
+ "valid": true,
1586
+ "error": null,
1587
+ "probs": {
1588
+ "yes": 0.1362,
1589
+ "no": 0.8638
1590
+ },
1591
+ "latency_s": 0.067
1592
+ },
1593
+ {
1594
+ "id": "original-adequacy-02-1",
1595
+ "tier": "standard",
1596
+ "expected": "no",
1597
+ "predicted": "no",
1598
+ "correct": true,
1599
+ "valid": true,
1600
+ "error": null,
1601
+ "probs": {
1602
+ "yes": 0.1605,
1603
+ "no": 0.8395
1604
+ },
1605
+ "latency_s": 0.067
1606
+ },
1607
+ {
1608
+ "id": "original-adequacy-03-0",
1609
+ "tier": "standard",
1610
+ "expected": "no",
1611
+ "predicted": "yes",
1612
+ "correct": false,
1613
+ "valid": true,
1614
+ "error": null,
1615
+ "probs": {
1616
+ "yes": 0.7873,
1617
+ "no": 0.2127
1618
+ },
1619
+ "latency_s": 0.068
1620
+ },
1621
+ {
1622
+ "id": "original-adequacy-03-1",
1623
+ "tier": "standard",
1624
+ "expected": "no",
1625
+ "predicted": "yes",
1626
+ "correct": false,
1627
+ "valid": true,
1628
+ "error": null,
1629
+ "probs": {
1630
+ "yes": 0.6999,
1631
+ "no": 0.3001
1632
+ },
1633
+ "latency_s": 0.068
1634
+ },
1635
+ {
1636
+ "id": "original-adequacy-04-0",
1637
+ "tier": "standard",
1638
+ "expected": "yes",
1639
+ "predicted": "yes",
1640
+ "correct": true,
1641
+ "valid": true,
1642
+ "error": null,
1643
+ "probs": {
1644
+ "yes": 0.8289,
1645
+ "no": 0.1711
1646
+ },
1647
+ "latency_s": 0.068
1648
+ },
1649
+ {
1650
+ "id": "original-adequacy-04-1",
1651
+ "tier": "standard",
1652
+ "expected": "yes",
1653
+ "predicted": "yes",
1654
+ "correct": true,
1655
+ "valid": true,
1656
+ "error": null,
1657
+ "probs": {
1658
+ "yes": 0.6751,
1659
+ "no": 0.3249
1660
+ },
1661
+ "latency_s": 0.068
1662
+ },
1663
+ {
1664
+ "id": "original-adequacy-05-0",
1665
+ "tier": "standard",
1666
+ "expected": "no",
1667
+ "predicted": "yes",
1668
+ "correct": false,
1669
+ "valid": true,
1670
+ "error": null,
1671
+ "probs": {
1672
+ "yes": 0.548,
1673
+ "no": 0.452
1674
+ },
1675
+ "latency_s": 0.068
1676
+ },
1677
+ {
1678
+ "id": "original-adequacy-05-1",
1679
+ "tier": "standard",
1680
+ "expected": "no",
1681
+ "predicted": "yes",
1682
+ "correct": false,
1683
+ "valid": true,
1684
+ "error": null,
1685
+ "probs": {
1686
+ "yes": 0.6917,
1687
+ "no": 0.3083
1688
+ },
1689
+ "latency_s": 0.068
1690
+ },
1691
+ {
1692
+ "id": "original-adequacy-06-0",
1693
+ "tier": "standard",
1694
+ "expected": "yes",
1695
+ "predicted": "yes",
1696
+ "correct": true,
1697
+ "valid": true,
1698
+ "error": null,
1699
+ "probs": {
1700
+ "yes": 0.548,
1701
+ "no": 0.452
1702
+ },
1703
+ "latency_s": 0.068
1704
+ },
1705
+ {
1706
+ "id": "original-adequacy-06-1",
1707
+ "tier": "standard",
1708
+ "expected": "yes",
1709
+ "predicted": "yes",
1710
+ "correct": true,
1711
+ "valid": true,
1712
+ "error": null,
1713
+ "probs": {
1714
+ "yes": 0.595,
1715
+ "no": 0.405
1716
+ },
1717
+ "latency_s": 0.07
1718
+ },
1719
+ {
1720
+ "id": "original-routing-01-0",
1721
+ "tier": "standard",
1722
+ "expected": "math",
1723
+ "predicted": "math",
1724
+ "correct": true,
1725
+ "valid": true,
1726
+ "error": null,
1727
+ "probs": {
1728
+ "coding": 0.0146,
1729
+ "coding_agent": 0.0019,
1730
+ "document": 0.0081,
1731
+ "general": 0.056,
1732
+ "math": 0.9138,
1733
+ "tools": 0.0056
1734
+ },
1735
+ "latency_s": 0.069
1736
+ },
1737
+ {
1738
+ "id": "original-routing-01-1",
1739
+ "tier": "standard",
1740
+ "expected": "math",
1741
+ "predicted": "math",
1742
+ "correct": true,
1743
+ "valid": true,
1744
+ "error": null,
1745
+ "probs": {
1746
+ "coding": 0.0124,
1747
+ "coding_agent": 0.0018,
1748
+ "document": 0.0106,
1749
+ "general": 0.059,
1750
+ "math": 0.9124,
1751
+ "tools": 0.0038
1752
+ },
1753
+ "latency_s": 0.07
1754
+ },
1755
+ {
1756
+ "id": "original-routing-02-0",
1757
+ "tier": "standard",
1758
+ "expected": "coding",
1759
+ "predicted": "coding",
1760
+ "correct": true,
1761
+ "valid": true,
1762
+ "error": null,
1763
+ "probs": {
1764
+ "coding": 0.847,
1765
+ "coding_agent": 0.0842,
1766
+ "document": 0.0057,
1767
+ "general": 0.0578,
1768
+ "math": 0.0023,
1769
+ "tools": 0.003
1770
+ },
1771
+ "latency_s": 0.068
1772
+ },
1773
+ {
1774
+ "id": "original-routing-02-1",
1775
+ "tier": "standard",
1776
+ "expected": "coding",
1777
+ "predicted": "coding",
1778
+ "correct": true,
1779
+ "valid": true,
1780
+ "error": null,
1781
+ "probs": {
1782
+ "coding": 0.8857,
1783
+ "coding_agent": 0.0462,
1784
+ "document": 0.0079,
1785
+ "general": 0.0543,
1786
+ "math": 0.0035,
1787
+ "tools": 0.0024
1788
+ },
1789
+ "latency_s": 0.067
1790
+ },
1791
+ {
1792
+ "id": "original-routing-03-0",
1793
+ "tier": "standard",
1794
+ "expected": "coding_agent",
1795
+ "predicted": "coding_agent",
1796
+ "correct": true,
1797
+ "valid": true,
1798
+ "error": null,
1799
+ "probs": {
1800
+ "coding": 0.0154,
1801
+ "coding_agent": 0.9638,
1802
+ "document": 0.0024,
1803
+ "general": 0.0131,
1804
+ "math": 0.0008,
1805
+ "tools": 0.0045
1806
+ },
1807
+ "latency_s": 0.068
1808
+ },
1809
+ {
1810
+ "id": "original-routing-03-1",
1811
+ "tier": "standard",
1812
+ "expected": "coding_agent",
1813
+ "predicted": "coding_agent",
1814
+ "correct": true,
1815
+ "valid": true,
1816
+ "error": null,
1817
+ "probs": {
1818
+ "coding": 0.009,
1819
+ "coding_agent": 0.9627,
1820
+ "document": 0.0029,
1821
+ "general": 0.0202,
1822
+ "math": 0.0007,
1823
+ "tools": 0.0045
1824
+ },
1825
+ "latency_s": 0.068
1826
+ },
1827
+ {
1828
+ "id": "original-routing-04-0",
1829
+ "tier": "standard",
1830
+ "expected": "document",
1831
+ "predicted": "document",
1832
+ "correct": true,
1833
+ "valid": true,
1834
+ "error": null,
1835
+ "probs": {
1836
+ "coding": 0.0085,
1837
+ "coding_agent": 0.0106,
1838
+ "document": 0.5328,
1839
+ "general": 0.4299,
1840
+ "math": 0.0019,
1841
+ "tools": 0.0163
1842
+ },
1843
+ "latency_s": 0.067
1844
+ },
1845
+ {
1846
+ "id": "original-routing-04-1",
1847
+ "tier": "standard",
1848
+ "expected": "document",
1849
+ "predicted": "document",
1850
+ "correct": true,
1851
+ "valid": true,
1852
+ "error": null,
1853
+ "probs": {
1854
+ "coding": 0.0053,
1855
+ "coding_agent": 0.005,
1856
+ "document": 0.782,
1857
+ "general": 0.1936,
1858
+ "math": 0.0016,
1859
+ "tools": 0.0125
1860
+ },
1861
+ "latency_s": 0.068
1862
+ },
1863
+ {
1864
+ "id": "original-routing-05-0",
1865
+ "tier": "standard",
1866
+ "expected": "tools",
1867
+ "predicted": "general",
1868
+ "correct": false,
1869
+ "valid": true,
1870
+ "error": null,
1871
+ "probs": {
1872
+ "coding": 0.0088,
1873
+ "coding_agent": 0.0041,
1874
+ "document": 0.0159,
1875
+ "general": 0.5494,
1876
+ "math": 0.0018,
1877
+ "tools": 0.42
1878
+ },
1879
+ "latency_s": 0.068
1880
+ },
1881
+ {
1882
+ "id": "original-routing-05-1",
1883
+ "tier": "standard",
1884
+ "expected": "tools",
1885
+ "predicted": "general",
1886
+ "correct": false,
1887
+ "valid": true,
1888
+ "error": null,
1889
+ "probs": {
1890
+ "coding": 0.0062,
1891
+ "coding_agent": 0.0031,
1892
+ "document": 0.0106,
1893
+ "general": 0.8663,
1894
+ "math": 0.0012,
1895
+ "tools": 0.1126
1896
+ },
1897
+ "latency_s": 0.068
1898
+ },
1899
+ {
1900
+ "id": "original-routing-06-0",
1901
+ "tier": "standard",
1902
+ "expected": "general",
1903
+ "predicted": "general",
1904
+ "correct": true,
1905
+ "valid": true,
1906
+ "error": null,
1907
+ "probs": {
1908
+ "coding": 0.0034,
1909
+ "coding_agent": 0.0019,
1910
+ "document": 0.0226,
1911
+ "general": 0.9684,
1912
+ "math": 0.0012,
1913
+ "tools": 0.0025
1914
+ },
1915
+ "latency_s": 0.068
1916
+ },
1917
+ {
1918
+ "id": "original-routing-06-1",
1919
+ "tier": "standard",
1920
+ "expected": "general",
1921
+ "predicted": "general",
1922
+ "correct": true,
1923
+ "valid": true,
1924
+ "error": null,
1925
+ "probs": {
1926
+ "coding": 0.004,
1927
+ "coding_agent": 0.002,
1928
+ "document": 0.025,
1929
+ "general": 0.9641,
1930
+ "math": 0.0012,
1931
+ "tools": 0.0036
1932
+ },
1933
+ "latency_s": 0.068
1934
+ },
1935
+ {
1936
+ "id": "hard-opus-a-long_policy-01",
1937
+ "tier": "hard",
1938
+ "expected": "pay_subject_to_15000_sublimit",
1939
+ "predicted": "pay_subject_to_15000_sublimit",
1940
+ "correct": true,
1941
+ "valid": true,
1942
+ "error": null,
1943
+ "probs": {
1944
+ "deny_repeated_seepage": 0.0068,
1945
+ "deny_vacancy_exclusion": 0.0152,
1946
+ "pay_full_estimate_less_deductible": 0.0044,
1947
+ "pay_subject_to_10000_sublimit": 0.0234,
1948
+ "pay_subject_to_15000_sublimit": 0.9502
1949
+ },
1950
+ "latency_s": 0.161
1951
+ },
1952
+ {
1953
+ "id": "hard-opus-a-long_policy-04",
1954
+ "tier": "hard",
1955
+ "expected": "cfo",
1956
+ "predicted": "vp_and_finance_director",
1957
+ "correct": false,
1958
+ "valid": true,
1959
+ "error": null,
1960
+ "probs": {
1961
+ "budget_holder": 0.0044,
1962
+ "cfo": 0.0044,
1963
+ "department_head": 0.3771,
1964
+ "executive_committee": 0.0028,
1965
+ "vp_and_finance_director": 0.6113
1966
+ },
1967
+ "latency_s": 0.095
1968
+ },
1969
+ {
1970
+ "id": "hard-opus-a-long_policy-08",
1971
+ "tier": "hard",
1972
+ "expected": "parts_covered_labour_charged",
1973
+ "predicted": "parts_covered_labour_charged",
1974
+ "correct": true,
1975
+ "valid": true,
1976
+ "error": null,
1977
+ "probs": {
1978
+ "full_repair_no_charge": 0.0483,
1979
+ "not_covered": 0.0194,
1980
+ "parts_covered_labour_charged": 0.9257,
1981
+ "pro_rata_replacement_credit": 0.0066
1982
+ },
1983
+ "latency_s": 0.086
1984
+ },
1985
+ {
1986
+ "id": "hard-opus-a-long_policy-09",
1987
+ "tier": "hard",
1988
+ "expected": "license_required",
1989
+ "predicted": "license_exception_lvs",
1990
+ "correct": false,
1991
+ "valid": true,
1992
+ "error": null,
1993
+ "probs": {
1994
+ "license_exception_lvs": 0.6411,
1995
+ "license_required": 0.219,
1996
+ "no_license_required": 0.1213,
1997
+ "shipment_prohibited": 0.0185
1998
+ },
1999
+ "latency_s": 0.088
2000
+ },
2001
+ {
2002
+ "id": "hard-opus-a-long_policy-11",
2003
+ "tier": "hard",
2004
+ "expected": "yes",
2005
+ "predicted": "no",
2006
+ "correct": false,
2007
+ "valid": true,
2008
+ "error": null,
2009
+ "probs": {
2010
+ "yes": 0.0705,
2011
+ "no": 0.9295
2012
+ },
2013
+ "latency_s": 0.083
2014
+ },
2015
+ {
2016
+ "id": "hard-opus-a-long_policy-13",
2017
+ "tier": "hard",
2018
+ "expected": "yes",
2019
+ "predicted": "no",
2020
+ "correct": false,
2021
+ "valid": true,
2022
+ "error": null,
2023
+ "probs": {
2024
+ "yes": 0.3596,
2025
+ "no": 0.6404
2026
+ },
2027
+ "latency_s": 0.085
2028
+ },
2029
+ {
2030
+ "id": "hard-opus-a-long_policy-17",
2031
+ "tier": "hard",
2032
+ "expected": "database_oncall",
2033
+ "predicted": "no_page_ticket_only",
2034
+ "correct": false,
2035
+ "valid": true,
2036
+ "error": null,
2037
+ "probs": {
2038
+ "database_oncall": 0.3601,
2039
+ "no_page_ticket_only": 0.5243,
2040
+ "platform_oncall": 0.0521,
2041
+ "security_incident_response": 0.0055,
2042
+ "vendor_support": 0.058
2043
+ },
2044
+ "latency_s": 0.083
2045
+ },
2046
+ {
2047
+ "id": "hard-opus-a-long_policy-19",
2048
+ "tier": "hard",
2049
+ "expected": "no",
2050
+ "predicted": "no",
2051
+ "correct": true,
2052
+ "valid": true,
2053
+ "error": null,
2054
+ "probs": {
2055
+ "yes": 0.2842,
2056
+ "no": 0.7158
2057
+ },
2058
+ "latency_s": 0.083
2059
+ },
2060
+ {
2061
+ "id": "hard-opus-a-probability-03",
2062
+ "tier": "hard",
2063
+ "expected": "yes",
2064
+ "predicted": "no",
2065
+ "correct": false,
2066
+ "valid": true,
2067
+ "error": null,
2068
+ "probs": {
2069
+ "yes": 0.4712,
2070
+ "no": 0.5288
2071
+ },
2072
+ "latency_s": 0.07
2073
+ },
2074
+ {
2075
+ "id": "hard-opus-a-probability-04",
2076
+ "tier": "hard",
2077
+ "expected": "on_time",
2078
+ "predicted": "on_time",
2079
+ "correct": true,
2080
+ "valid": true,
2081
+ "error": null,
2082
+ "probs": {
2083
+ "early": 0.0373,
2084
+ "late": 0.208,
2085
+ "on_time": 0.7547
2086
+ },
2087
+ "latency_s": 0.071
2088
+ },
2089
+ {
2090
+ "id": "hard-opus-a-probability-07",
2091
+ "tier": "hard",
2092
+ "expected": "escalated",
2093
+ "predicted": "escalated",
2094
+ "correct": true,
2095
+ "valid": true,
2096
+ "error": null,
2097
+ "probs": {
2098
+ "escalated": 0.7725,
2099
+ "reopened": 0.0362,
2100
+ "resolved_first_contact": 0.1913
2101
+ },
2102
+ "latency_s": 0.071
2103
+ },
2104
+ {
2105
+ "id": "hard-opus-a-probability-08",
2106
+ "tier": "hard",
2107
+ "expected": "no",
2108
+ "predicted": "no",
2109
+ "correct": true,
2110
+ "valid": true,
2111
+ "error": null,
2112
+ "probs": {
2113
+ "yes": 0.405,
2114
+ "no": 0.595
2115
+ },
2116
+ "latency_s": 0.069
2117
+ },
2118
+ {
2119
+ "id": "hard-opus-a-temporal_numeric-03",
2120
+ "tier": "hard",
2121
+ "expected": "yes",
2122
+ "predicted": "no",
2123
+ "correct": false,
2124
+ "valid": true,
2125
+ "error": null,
2126
+ "probs": {
2127
+ "yes": 0.3249,
2128
+ "no": 0.6751
2129
+ },
2130
+ "latency_s": 0.069
2131
+ },
2132
+ {
2133
+ "id": "hard-opus-a-temporal_numeric-06",
2134
+ "tier": "hard",
2135
+ "expected": "no",
2136
+ "predicted": "no",
2137
+ "correct": true,
2138
+ "valid": true,
2139
+ "error": null,
2140
+ "probs": {
2141
+ "yes": 0.3507,
2142
+ "no": 0.6493
2143
+ },
2144
+ "latency_s": 0.069
2145
+ },
2146
+ {
2147
+ "id": "hard-opus-a-temporal_numeric-07",
2148
+ "tier": "hard",
2149
+ "expected": "sep_26",
2150
+ "predicted": "sep_28",
2151
+ "correct": false,
2152
+ "valid": true,
2153
+ "error": null,
2154
+ "probs": {
2155
+ "sep_24": 0.4281,
2156
+ "sep_25": 0.0403,
2157
+ "sep_26": 0.0212,
2158
+ "sep_27": 0.0587,
2159
+ "sep_28": 0.4517
2160
+ },
2161
+ "latency_s": 0.073
2162
+ },
2163
+ {
2164
+ "id": "hard-opus-a-temporal_numeric-09",
2165
+ "tier": "hard",
2166
+ "expected": "no",
2167
+ "predicted": "yes",
2168
+ "correct": false,
2169
+ "valid": true,
2170
+ "error": null,
2171
+ "probs": {
2172
+ "yes": 0.5669,
2173
+ "no": 0.4331
2174
+ },
2175
+ "latency_s": 0.069
2176
+ },
2177
+ {
2178
+ "id": "hard-opus-a-temporal_numeric-12",
2179
+ "tier": "hard",
2180
+ "expected": "1",
2181
+ "predicted": "2",
2182
+ "correct": false,
2183
+ "valid": true,
2184
+ "error": null,
2185
+ "probs": {
2186
+ "0": 0.1753,
2187
+ "1": 0.3425,
2188
+ "2": 0.4708,
2189
+ "3": 0.0114
2190
+ },
2191
+ "latency_s": 0.075
2192
+ },
2193
+ {
2194
+ "id": "hard-opus-b-ambiguous-02",
2195
+ "tier": "hard",
2196
+ "expected": "real_incident",
2197
+ "predicted": "false_positive",
2198
+ "correct": false,
2199
+ "valid": true,
2200
+ "error": null,
2201
+ "probs": {
2202
+ "cannot_determine": 0.0626,
2203
+ "false_positive": 0.4813,
2204
+ "real_incident": 0.4561
2205
+ },
2206
+ "latency_s": 0.07
2207
+ },
2208
+ {
2209
+ "id": "hard-opus-b-ambiguous-03",
2210
+ "tier": "hard",
2211
+ "expected": "time_barred",
2212
+ "predicted": "time_barred",
2213
+ "correct": true,
2214
+ "valid": true,
2215
+ "error": null,
2216
+ "probs": {
2217
+ "insufficient_information": 0.0281,
2218
+ "time_barred": 0.538,
2219
+ "within_limitation": 0.434
2220
+ },
2221
+ "latency_s": 0.068
2222
+ },
2223
+ {
2224
+ "id": "hard-opus-b-ambiguous-07",
2225
+ "tier": "hard",
2226
+ "expected": "within_policy",
2227
+ "predicted": "insufficient_information",
2228
+ "correct": false,
2229
+ "valid": true,
2230
+ "error": null,
2231
+ "probs": {
2232
+ "insufficient_information": 0.7665,
2233
+ "over_limit": 0.1531,
2234
+ "within_policy": 0.0804
2235
+ },
2236
+ "latency_s": 0.068
2237
+ },
2238
+ {
2239
+ "id": "hard-opus-b-ambiguous-09",
2240
+ "tier": "hard",
2241
+ "expected": "legitimate",
2242
+ "predicted": "compromised",
2243
+ "correct": false,
2244
+ "valid": true,
2245
+ "error": null,
2246
+ "probs": {
2247
+ "compromised": 0.5581,
2248
+ "inconclusive": 0.0153,
2249
+ "legitimate": 0.4267
2250
+ },
2251
+ "latency_s": 0.07
2252
+ },
2253
+ {
2254
+ "id": "hard-opus-b-ambiguous-10",
2255
+ "tier": "hard",
2256
+ "expected": "ask_user_which_alex",
2257
+ "predicted": "send_to_alex_moreno",
2258
+ "correct": false,
2259
+ "valid": true,
2260
+ "error": null,
2261
+ "probs": {
2262
+ "ask_user_which_alex": 0.2108,
2263
+ "send_to_alex_moreno": 0.7646,
2264
+ "send_to_alexandra_lind": 0.0246
2265
+ },
2266
+ "latency_s": 0.068
2267
+ },
2268
+ {
2269
+ "id": "hard-opus-b-ambiguous-11",
2270
+ "tier": "hard",
2271
+ "expected": "cannot_determine",
2272
+ "predicted": "cannot_determine",
2273
+ "correct": true,
2274
+ "valid": true,
2275
+ "error": null,
2276
+ "probs": {
2277
+ "cannot_determine": 0.9775,
2278
+ "grade_correct": 0.0174,
2279
+ "grade_should_be_higher": 0.0051
2280
+ },
2281
+ "latency_s": 0.068
2282
+ },
2283
+ {
2284
+ "id": "hard-opus-b-ambiguous-13",
2285
+ "tier": "hard",
2286
+ "expected": "cannot_determine",
2287
+ "predicted": "cannot_determine",
2288
+ "correct": true,
2289
+ "valid": true,
2290
+ "error": null,
2291
+ "probs": {
2292
+ "cannot_determine": 0.9869,
2293
+ "charge_data_team": 0.0023,
2294
+ "charge_platform_team": 0.0051,
2295
+ "split_50_50": 0.0057
2296
+ },
2297
+ "latency_s": 0.069
2298
+ },
2299
+ {
2300
+ "id": "hard-opus-b-multi_hop-03",
2301
+ "tier": "hard",
2302
+ "expected": "tier3_cfo",
2303
+ "predicted": "tier3_cfo",
2304
+ "correct": true,
2305
+ "valid": true,
2306
+ "error": null,
2307
+ "probs": {
2308
+ "tier1_team_lead": 0.1383,
2309
+ "tier2_department_head": 0.1383,
2310
+ "tier3_cfo": 0.6926,
2311
+ "tier4_board": 0.0308
2312
+ },
2313
+ "latency_s": 0.093
2314
+ },
2315
+ {
2316
+ "id": "hard-opus-b-multi_hop-04",
2317
+ "tier": "hard",
2318
+ "expected": "bjorn",
2319
+ "predicted": "no_page_ticket_only",
2320
+ "correct": false,
2321
+ "valid": true,
2322
+ "error": null,
2323
+ "probs": {
2324
+ "ana": 0.0277,
2325
+ "bjorn": 0.0263,
2326
+ "chen": 0.1183,
2327
+ "dmitri": 0.0053,
2328
+ "esra": 0.0053,
2329
+ "no_page_ticket_only": 0.8172
2330
+ },
2331
+ "latency_s": 0.082
2332
+ },
2333
+ {
2334
+ "id": "hard-opus-b-multi_hop-05",
2335
+ "tier": "hard",
2336
+ "expected": "2",
2337
+ "predicted": "2",
2338
+ "correct": true,
2339
+ "valid": true,
2340
+ "error": null,
2341
+ "probs": {
2342
+ "0": 0.0211,
2343
+ "1": 0.2058,
2344
+ "2": 0.6362,
2345
+ "3": 0.0922,
2346
+ "4": 0.0447
2347
+ },
2348
+ "latency_s": 11.645
2349
+ },
2350
+ {
2351
+ "id": "hard-opus-b-multi_hop-07",
2352
+ "tier": "hard",
2353
+ "expected": "credit_20_percent",
2354
+ "predicted": "credit_50_percent",
2355
+ "correct": false,
2356
+ "valid": true,
2357
+ "error": null,
2358
+ "probs": {
2359
+ "credit_10_percent": 0.0453,
2360
+ "credit_20_percent": 0.0504,
2361
+ "credit_25_percent": 0.3486,
2362
+ "credit_50_percent": 0.4811,
2363
+ "credit_full_month": 0.0532,
2364
+ "no_credit": 0.0214
2365
+ },
2366
+ "latency_s": 0.091
2367
+ },
2368
+ {
2369
+ "id": "hard-opus-b-multi_hop-08",
2370
+ "tier": "hard",
2371
+ "expected": "no",
2372
+ "predicted": "no",
2373
+ "correct": true,
2374
+ "valid": true,
2375
+ "error": null,
2376
+ "probs": {
2377
+ "yes": 0.2921,
2378
+ "no": 0.7079
2379
+ },
2380
+ "latency_s": 0.079
2381
+ },
2382
+ {
2383
+ "id": "hard-opus-b-probability-01",
2384
+ "tier": "hard",
2385
+ "expected": "merchant_wins",
2386
+ "predicted": "merchant_wins",
2387
+ "correct": true,
2388
+ "valid": true,
2389
+ "error": null,
2390
+ "probs": {
2391
+ "customer_wins": 0.0037,
2392
+ "customer_withdraws": 0.0057,
2393
+ "merchant_wins": 0.9906
2394
+ },
2395
+ "latency_s": 0.075
2396
+ },
2397
+ {
2398
+ "id": "hard-opus-b-probability-02",
2399
+ "tier": "hard",
2400
+ "expected": "upstream_provider",
2401
+ "predicted": "bad_push",
2402
+ "correct": false,
2403
+ "valid": true,
2404
+ "error": null,
2405
+ "probs": {
2406
+ "bad_push": 0.6836,
2407
+ "database_hardware": 0.0109,
2408
+ "upstream_provider": 0.3055
2409
+ },
2410
+ "latency_s": 0.068
2411
+ },
2412
+ {
2413
+ "id": "hard-opus-b-probability-03",
2414
+ "tier": "hard",
2415
+ "expected": "yes",
2416
+ "predicted": "yes",
2417
+ "correct": true,
2418
+ "valid": true,
2419
+ "error": null,
2420
+ "probs": {
2421
+ "yes": 0.6666,
2422
+ "no": 0.3334
2423
+ },
2424
+ "latency_s": 0.071
2425
+ },
2426
+ {
2427
+ "id": "hard-opus-b-probability-04",
2428
+ "tier": "hard",
2429
+ "expected": "attended",
2430
+ "predicted": "attended",
2431
+ "correct": true,
2432
+ "valid": true,
2433
+ "error": null,
2434
+ "probs": {
2435
+ "attended": 0.7619,
2436
+ "late_cancellation": 0.0939,
2437
+ "no_show": 0.1442
2438
+ },
2439
+ "latency_s": 0.069
2440
+ },
2441
+ {
2442
+ "id": "hard-opus-b-probability-06",
2443
+ "tier": "hard",
2444
+ "expected": "upheld",
2445
+ "predicted": "upheld",
2446
+ "correct": true,
2447
+ "valid": true,
2448
+ "error": null,
2449
+ "probs": {
2450
+ "modified": 0.0265,
2451
+ "overturned": 0.0562,
2452
+ "upheld": 0.9173
2453
+ },
2454
+ "latency_s": 0.072
2455
+ },
2456
+ {
2457
+ "id": "hard-opus-b-tradeoff-01",
2458
+ "tier": "hard",
2459
+ "expected": "escalate_to_engineering",
2460
+ "predicted": "escalate_to_security",
2461
+ "correct": false,
2462
+ "valid": true,
2463
+ "error": null,
2464
+ "probs": {
2465
+ "escalate_to_engineering": 0.2134,
2466
+ "escalate_to_security": 0.7741,
2467
+ "page_oncall": 0.0045,
2468
+ "process_refund": 0.0062,
2469
+ "send_kb_article": 0.0019
2470
+ },
2471
+ "latency_s": 0.072
2472
+ },
2473
+ {
2474
+ "id": "hard-opus-b-tradeoff-03",
2475
+ "tier": "hard",
2476
+ "expected": "order_b",
2477
+ "predicted": "order_b",
2478
+ "correct": true,
2479
+ "valid": true,
2480
+ "error": null,
2481
+ "probs": {
2482
+ "order_a": 0.0518,
2483
+ "order_b": 0.892,
2484
+ "order_c": 0.0187,
2485
+ "order_d": 0.0375
2486
+ },
2487
+ "latency_s": 0.071
2488
+ },
2489
+ {
2490
+ "id": "hard-opus-b-tradeoff-06",
2491
+ "tier": "hard",
2492
+ "expected": "yes",
2493
+ "predicted": "no",
2494
+ "correct": false,
2495
+ "valid": true,
2496
+ "error": null,
2497
+ "probs": {
2498
+ "yes": 0.1455,
2499
+ "no": 0.8545
2500
+ },
2501
+ "latency_s": 0.07
2502
+ },
2503
+ {
2504
+ "id": "hard-opus-b-tradeoff-07",
2505
+ "tier": "hard",
2506
+ "expected": "p0_fix_24h",
2507
+ "predicted": "p0_fix_24h",
2508
+ "correct": true,
2509
+ "valid": true,
2510
+ "error": null,
2511
+ "probs": {
2512
+ "out_of_scope": 0.0022,
2513
+ "p0_fix_24h": 0.546,
2514
+ "p1_fix_7d": 0.2192,
2515
+ "p2_fix_30d": 0.2313,
2516
+ "p3_backlog": 0.0014
2517
+ },
2518
+ "latency_s": 0.071
2519
+ },
2520
+ {
2521
+ "id": "hard-opus-b-tradeoff-08",
2522
+ "tier": "hard",
2523
+ "expected": "yes",
2524
+ "predicted": "no",
2525
+ "correct": false,
2526
+ "valid": true,
2527
+ "error": null,
2528
+ "probs": {
2529
+ "yes": 0.1553,
2530
+ "no": 0.8447
2531
+ },
2532
+ "latency_s": 0.07
2533
+ },
2534
+ {
2535
+ "id": "hard-opus-b-tradeoff-12",
2536
+ "tier": "hard",
2537
+ "expected": "refund_now",
2538
+ "predicted": "refund_now",
2539
+ "correct": true,
2540
+ "valid": true,
2541
+ "error": null,
2542
+ "probs": {
2543
+ "deny_refund": 0.0147,
2544
+ "escalate_to_fraud_team": 0.0015,
2545
+ "offer_store_credit": 0.0048,
2546
+ "refund_now": 0.9689,
2547
+ "wait_for_inspection": 0.0101
2548
+ },
2549
+ "latency_s": 0.071
2550
+ },
2551
+ {
2552
+ "id": "hard-opus-c-long_policy-02",
2553
+ "tier": "hard",
2554
+ "expected": "erase_but_retain_transaction_records",
2555
+ "predicted": "restrict_all_pending_legal_hold",
2556
+ "correct": false,
2557
+ "valid": true,
2558
+ "error": null,
2559
+ "probs": {
2560
+ "erase_all": 0.0076,
2561
+ "erase_but_retain_transaction_records": 0.2925,
2562
+ "erase_marketing_only": 0.0055,
2563
+ "reject_unverified_request": 0.0038,
2564
+ "restrict_all_pending_legal_hold": 0.6906
2565
+ },
2566
+ "latency_s": 0.123
2567
+ },
2568
+ {
2569
+ "id": "hard-opus-c-long_policy-03",
2570
+ "tier": "hard",
2571
+ "expected": "eur_400",
2572
+ "predicted": "eur_0",
2573
+ "correct": false,
2574
+ "valid": true,
2575
+ "error": null,
2576
+ "probs": {
2577
+ "eur_0": 0.3147,
2578
+ "eur_150": 0.07,
2579
+ "eur_200": 0.228,
2580
+ "eur_300": 0.1566,
2581
+ "eur_400": 0.1941,
2582
+ "eur_600": 0.0367
2583
+ },
2584
+ "latency_s": 0.122
2585
+ },
2586
+ {
2587
+ "id": "hard-opus-c-long_policy-04",
2588
+ "tier": "hard",
2589
+ "expected": "yes",
2590
+ "predicted": "yes",
2591
+ "correct": true,
2592
+ "valid": true,
2593
+ "error": null,
2594
+ "probs": {
2595
+ "yes": 0.6043,
2596
+ "no": 0.3957
2597
+ },
2598
+ "latency_s": 0.108
2599
+ },
2600
+ {
2601
+ "id": "hard-opus-c-long_policy-05",
2602
+ "tier": "hard",
2603
+ "expected": "1",
2604
+ "predicted": "2",
2605
+ "correct": false,
2606
+ "valid": true,
2607
+ "error": null,
2608
+ "probs": {
2609
+ "0": 0.1075,
2610
+ "1": 0.3735,
2611
+ "2": 0.4746,
2612
+ "3": 0.0444
2613
+ },
2614
+ "latency_s": 0.384
2615
+ },
2616
+ {
2617
+ "id": "hard-opus-c-long_policy-08",
2618
+ "tier": "hard",
2619
+ "expected": "1",
2620
+ "predicted": "1",
2621
+ "correct": true,
2622
+ "valid": true,
2623
+ "error": null,
2624
+ "probs": {
2625
+ "0": 0.0124,
2626
+ "1": 0.773,
2627
+ "2": 0.1642,
2628
+ "3": 0.0503
2629
+ },
2630
+ "latency_s": 0.391
2631
+ },
2632
+ {
2633
+ "id": "hard-opus-c-long_policy-10",
2634
+ "tier": "hard",
2635
+ "expected": "credit_minus_restocking_fee",
2636
+ "predicted": "credit_minus_restocking_fee",
2637
+ "correct": true,
2638
+ "valid": true,
2639
+ "error": null,
2640
+ "probs": {
2641
+ "credit_minus_restocking_fee": 0.6196,
2642
+ "full_credit": 0.1454,
2643
+ "reject_return": 0.0897,
2644
+ "repair_or_replace_only": 0.1454
2645
+ },
2646
+ "latency_s": 0.126
2647
+ },
2648
+ {
2649
+ "id": "hard-opus-c-long_policy-11",
2650
+ "tier": "hard",
2651
+ "expected": "no",
2652
+ "predicted": "no",
2653
+ "correct": true,
2654
+ "valid": true,
2655
+ "error": null,
2656
+ "probs": {
2657
+ "yes": 0.2467,
2658
+ "no": 0.7533
2659
+ },
2660
+ "latency_s": 0.109
2661
+ },
2662
+ {
2663
+ "id": "hard-opus-c-probability-03",
2664
+ "tier": "hard",
2665
+ "expected": "kuznets_technik",
2666
+ "predicted": "halvorsen_polymer",
2667
+ "correct": false,
2668
+ "valid": true,
2669
+ "error": null,
2670
+ "probs": {
2671
+ "halvorsen_polymer": 0.8857,
2672
+ "kuznets_technik": 0.0393,
2673
+ "ortega_seal_works": 0.0749
2674
+ },
2675
+ "latency_s": 0.07
2676
+ },
2677
+ {
2678
+ "id": "hard-opus-c-temporal_numeric-02",
2679
+ "tier": "hard",
2680
+ "expected": "yes",
2681
+ "predicted": "no",
2682
+ "correct": false,
2683
+ "valid": true,
2684
+ "error": null,
2685
+ "probs": {
2686
+ "yes": 0.1823,
2687
+ "no": 0.8177
2688
+ },
2689
+ "latency_s": 0.071
2690
+ },
2691
+ {
2692
+ "id": "hard-opus-c-temporal_numeric-03",
2693
+ "tier": "hard",
2694
+ "expected": "eur_692_00",
2695
+ "predicted": "eur_693_90",
2696
+ "correct": false,
2697
+ "valid": true,
2698
+ "error": null,
2699
+ "probs": {
2700
+ "eur_617_00": 0.0276,
2701
+ "eur_688_00": 0.0852,
2702
+ "eur_692_00": 0.326,
2703
+ "eur_693_90": 0.5577,
2704
+ "eur_812_00": 0.0036
2705
+ },
2706
+ "latency_s": 0.077
2707
+ },
2708
+ {
2709
+ "id": "hard-opus-c-temporal_numeric-04",
2710
+ "tier": "hard",
2711
+ "expected": "usd_2335_00",
2712
+ "predicted": "usd_2303_01",
2713
+ "correct": false,
2714
+ "valid": true,
2715
+ "error": null,
2716
+ "probs": {
2717
+ "usd_2222_50": 0.0252,
2718
+ "usd_2303_01": 0.4331,
2719
+ "usd_2303_33": 0.4105,
2720
+ "usd_2335_00": 0.1016,
2721
+ "usd_2340_00": 0.0296
2722
+ },
2723
+ "latency_s": 0.072
2724
+ },
2725
+ {
2726
+ "id": "hard-opus-c-temporal_numeric-06",
2727
+ "tier": "hard",
2728
+ "expected": "sep_18_book_set",
2729
+ "predicted": "none_declined",
2730
+ "correct": false,
2731
+ "valid": true,
2732
+ "error": null,
2733
+ "probs": {
2734
+ "none_declined": 0.7695,
2735
+ "sep_15_office_chair": 0.0651,
2736
+ "sep_16_taxi": 0.0342,
2737
+ "sep_17_parking": 0.0948,
2738
+ "sep_18_book_set": 0.0235,
2739
+ "sep_21_client_dinner": 0.013
2740
+ },
2741
+ "latency_s": 0.071
2742
+ },
2743
+ {
2744
+ "id": "hard-opus-c-temporal_numeric-08",
2745
+ "tier": "hard",
2746
+ "expected": "eur_498_81",
2747
+ "predicted": "eur_513_69",
2748
+ "correct": false,
2749
+ "valid": true,
2750
+ "error": null,
2751
+ "probs": {
2752
+ "eur_497_40": 0.0424,
2753
+ "eur_498_81": 0.1057,
2754
+ "eur_506_06": 0.3443,
2755
+ "eur_513_69": 0.4268,
2756
+ "eur_532_01": 0.0808
2757
+ },
2758
+ "latency_s": 0.071
2759
+ },
2760
+ {
2761
+ "id": "hard-opus-c-temporal_numeric-12",
2762
+ "tier": "hard",
2763
+ "expected": "expired_30_month_cap",
2764
+ "predicted": "expired_30_month_cap",
2765
+ "correct": true,
2766
+ "valid": true,
2767
+ "error": null,
2768
+ "probs": {
2769
+ "covered": 0.0365,
2770
+ "expired_24_month_term": 0.096,
2771
+ "expired_30_month_cap": 0.8675
2772
+ },
2773
+ "latency_s": 0.071
2774
+ },
2775
+ {
2776
+ "id": "hard-sol-a-adversarial-01",
2777
+ "tier": "hard",
2778
+ "expected": "deny_missing_approval",
2779
+ "predicted": "deny_missing_approval",
2780
+ "correct": true,
2781
+ "valid": true,
2782
+ "error": null,
2783
+ "probs": {
2784
+ "deny_inactive_user": 0.001,
2785
+ "deny_missing_approval": 0.9798,
2786
+ "grant_access": 0.0043,
2787
+ "grant_owner_exception": 0.0149
2788
+ },
2789
+ "latency_s": 0.069
2790
+ },
2791
+ {
2792
+ "id": "hard-sol-a-adversarial-06",
2793
+ "tier": "hard",
2794
+ "expected": "no",
2795
+ "predicted": "no",
2796
+ "correct": true,
2797
+ "valid": true,
2798
+ "error": null,
2799
+ "probs": {
2800
+ "yes": 0.0252,
2801
+ "no": 0.9748
2802
+ },
2803
+ "latency_s": 0.069
2804
+ },
2805
+ {
2806
+ "id": "hard-sol-a-adversarial-07",
2807
+ "tier": "hard",
2808
+ "expected": "retain_until_2031",
2809
+ "predicted": "retain_until_2031",
2810
+ "correct": true,
2811
+ "valid": true,
2812
+ "error": null,
2813
+ "probs": {
2814
+ "delete_now": 0.0328,
2815
+ "needs_signature_review": 0.0007,
2816
+ "retain_indefinitely": 0.0014,
2817
+ "retain_until_2031": 0.9651
2818
+ },
2819
+ "latency_s": 0.069
2820
+ },
2821
+ {
2822
+ "id": "hard-sol-a-adversarial-08",
2823
+ "tier": "hard",
2824
+ "expected": "yes",
2825
+ "predicted": "yes",
2826
+ "correct": true,
2827
+ "valid": true,
2828
+ "error": null,
2829
+ "probs": {
2830
+ "yes": 0.9757,
2831
+ "no": 0.0243
2832
+ },
2833
+ "latency_s": 0.068
2834
+ },
2835
+ {
2836
+ "id": "hard-sol-a-adversarial-09",
2837
+ "tier": "hard",
2838
+ "expected": "2",
2839
+ "predicted": "2",
2840
+ "correct": true,
2841
+ "valid": true,
2842
+ "error": null,
2843
+ "probs": {
2844
+ "0": 0.0125,
2845
+ "1": 0.052,
2846
+ "2": 0.7757,
2847
+ "3": 0.1598
2848
+ },
2849
+ "latency_s": 0.071
2850
+ },
2851
+ {
2852
+ "id": "hard-sol-a-adversarial-11",
2853
+ "tier": "hard",
2854
+ "expected": "standard_research",
2855
+ "predicted": "standard_research",
2856
+ "correct": true,
2857
+ "valid": true,
2858
+ "error": null,
2859
+ "probs": {
2860
+ "ethics_review": 0.326,
2861
+ "major_grants": 0.0145,
2862
+ "manual_hold": 0.0042,
2863
+ "standard_research": 0.6553
2864
+ },
2865
+ "latency_s": 0.069
2866
+ },
2867
+ {
2868
+ "id": "hard-sol-a-multi_hop-01",
2869
+ "tier": "hard",
2870
+ "expected": "accept_s2_dusk",
2871
+ "predicted": "reject_cutoff",
2872
+ "correct": false,
2873
+ "valid": true,
2874
+ "error": null,
2875
+ "probs": {
2876
+ "accept_s2_dusk": 0.2306,
2877
+ "reject_cutoff": 0.6061,
2878
+ "reject_tier": 0.1147,
2879
+ "reroute_s3": 0.0486
2880
+ },
2881
+ "latency_s": 0.086
2882
+ },
2883
+ {
2884
+ "id": "hard-sol-a-multi_hop-02",
2885
+ "tier": "hard",
2886
+ "expected": "security_primary",
2887
+ "predicted": "security_primary",
2888
+ "correct": true,
2889
+ "valid": true,
2890
+ "error": null,
2891
+ "probs": {
2892
+ "executive_bridge": 0.23,
2893
+ "heron_q2": 0.0669,
2894
+ "insufficient_information": 0.0299,
2895
+ "security_primary": 0.6732
2896
+ },
2897
+ "latency_s": 0.086
2898
+ },
2899
+ {
2900
+ "id": "hard-sol-a-multi_hop-05",
2901
+ "tier": "hard",
2902
+ "expected": "require_director",
2903
+ "predicted": "require_director",
2904
+ "correct": true,
2905
+ "valid": true,
2906
+ "error": null,
2907
+ "probs": {
2908
+ "pay_manager_approved": 0.0378,
2909
+ "reject_category": 0.2345,
2910
+ "require_director": 0.7242,
2911
+ "require_finance_board": 0.0036
2912
+ },
2913
+ "latency_s": 0.084
2914
+ },
2915
+ {
2916
+ "id": "hard-sol-a-multi_hop-07",
2917
+ "tier": "hard",
2918
+ "expected": "admit",
2919
+ "predicted": "deny_prerequisite",
2920
+ "correct": false,
2921
+ "valid": true,
2922
+ "error": null,
2923
+ "probs": {
2924
+ "admit": 0.1503,
2925
+ "deny_prerequisite": 0.5755,
2926
+ "deny_score": 0.0879,
2927
+ "manual_equivalency": 0.1863
2928
+ },
2929
+ "latency_s": 0.084
2930
+ },
2931
+ {
2932
+ "id": "hard-sol-a-multi_hop-08",
2933
+ "tier": "hard",
2934
+ "expected": "central_engineering",
2935
+ "predicted": "central_engineering",
2936
+ "correct": true,
2937
+ "valid": true,
2938
+ "error": null,
2939
+ "probs": {
2940
+ "central_engineering": 0.8315,
2941
+ "insufficient_log": 0.014,
2942
+ "local_maintenance": 0.1492,
2943
+ "safety_shutdown": 0.0053
2944
+ },
2945
+ "latency_s": 0.084
2946
+ },
2947
+ {
2948
+ "id": "hard-sol-a-multi_hop-09",
2949
+ "tier": "hard",
2950
+ "expected": "require_steward",
2951
+ "predicted": "require_steward",
2952
+ "correct": true,
2953
+ "valid": true,
2954
+ "error": null,
2955
+ "probs": {
2956
+ "confirm_l3": 0.0057,
2957
+ "reject_capacity": 0.0097,
2958
+ "require_steward": 0.983,
2959
+ "switch_l4": 0.0017
2960
+ },
2961
+ "latency_s": 0.085
2962
+ },
2963
+ {
2964
+ "id": "hard-sol-a-multi_hop-10",
2965
+ "tier": "hard",
2966
+ "expected": "approve_45_days",
2967
+ "predicted": "reduce_to_30_days",
2968
+ "correct": false,
2969
+ "valid": true,
2970
+ "error": null,
2971
+ "probs": {
2972
+ "approve_45_days": 0.4122,
2973
+ "deny_sensitive": 0.1335,
2974
+ "needs_field_review": 0.0193,
2975
+ "reduce_to_30_days": 0.435
2976
+ },
2977
+ "latency_s": 0.085
2978
+ },
2979
+ {
2980
+ "id": "hard-sol-a-multi_hop-12",
2981
+ "tier": "hard",
2982
+ "expected": "needs_authentication",
2983
+ "predicted": "covered_recurrence",
2984
+ "correct": false,
2985
+ "valid": true,
2986
+ "error": null,
2987
+ "probs": {
2988
+ "covered_recurrence": 0.6299,
2989
+ "expired": 0.0626,
2990
+ "needs_authentication": 0.2668,
2991
+ "wrong_issue_class": 0.0407
2992
+ },
2993
+ "latency_s": 0.085
2994
+ },
2995
+ {
2996
+ "id": "hard-sol-a-trap-02",
2997
+ "tier": "hard",
2998
+ "expected": "no",
2999
+ "predicted": "no",
3000
+ "correct": true,
3001
+ "valid": true,
3002
+ "error": null,
3003
+ "probs": {
3004
+ "yes": 0.0098,
3005
+ "no": 0.9902
3006
+ },
3007
+ "latency_s": 0.069
3008
+ },
3009
+ {
3010
+ "id": "hard-sol-a-trap-04",
3011
+ "tier": "hard",
3012
+ "expected": "no",
3013
+ "predicted": "no",
3014
+ "correct": true,
3015
+ "valid": true,
3016
+ "error": null,
3017
+ "probs": {
3018
+ "yes": 0.0813,
3019
+ "no": 0.9187
3020
+ },
3021
+ "latency_s": 0.069
3022
+ },
3023
+ {
3024
+ "id": "hard-sol-a-trap-05",
3025
+ "tier": "hard",
3026
+ "expected": "0",
3027
+ "predicted": "0",
3028
+ "correct": true,
3029
+ "valid": true,
3030
+ "error": null,
3031
+ "probs": {
3032
+ "0": 0.5884,
3033
+ "1": 0.0304,
3034
+ "2": 0.3742,
3035
+ "3": 0.007
3036
+ },
3037
+ "latency_s": 0.074
3038
+ },
3039
+ {
3040
+ "id": "hard-sol-a-trap-06",
3041
+ "tier": "hard",
3042
+ "expected": "keep_open_unresolved",
3043
+ "predicted": "keep_open_unresolved",
3044
+ "correct": true,
3045
+ "valid": true,
3046
+ "error": null,
3047
+ "probs": {
3048
+ "await_internal_metrics": 0.0091,
3049
+ "close_agent_verified": 0.002,
3050
+ "close_confirmed": 0.0183,
3051
+ "keep_open_unresolved": 0.9706
3052
+ },
3053
+ "latency_s": 0.069
3054
+ },
3055
+ {
3056
+ "id": "hard-sol-a-trap-08",
3057
+ "tier": "hard",
3058
+ "expected": "late",
3059
+ "predicted": "late",
3060
+ "correct": true,
3061
+ "valid": true,
3062
+ "error": null,
3063
+ "probs": {
3064
+ "canceled": 0.0002,
3065
+ "indeterminate": 0.0003,
3066
+ "late": 0.979,
3067
+ "on_time": 0.0205
3068
+ },
3069
+ "latency_s": 0.069
3070
+ },
3071
+ {
3072
+ "id": "hard-sol-a-trap-10",
3073
+ "tier": "hard",
3074
+ "expected": "blocked_second_review",
3075
+ "predicted": "blocked_second_review",
3076
+ "correct": true,
3077
+ "valid": true,
3078
+ "error": null,
3079
+ "probs": {
3080
+ "blocked_failed_test": 0.0041,
3081
+ "blocked_second_review": 0.9297,
3082
+ "ready_one_review": 0.0092,
3083
+ "ready_two_reviews": 0.057
3084
+ },
3085
+ "latency_s": 0.068
3086
+ },
3087
+ {
3088
+ "id": "hard-sol-a-trap-13",
3089
+ "tier": "hard",
3090
+ "expected": "no",
3091
+ "predicted": "no",
3092
+ "correct": true,
3093
+ "valid": true,
3094
+ "error": null,
3095
+ "probs": {
3096
+ "yes": 0.044,
3097
+ "no": 0.956
3098
+ },
3099
+ "latency_s": 0.068
3100
+ },
3101
+ {
3102
+ "id": "hard-sol-a-trap-15",
3103
+ "tier": "hard",
3104
+ "expected": "no",
3105
+ "predicted": "no",
3106
+ "correct": true,
3107
+ "valid": true,
3108
+ "error": null,
3109
+ "probs": {
3110
+ "yes": 0.3596,
3111
+ "no": 0.6404
3112
+ },
3113
+ "latency_s": 0.068
3114
+ },
3115
+ {
3116
+ "id": "hard-sol-b-judge_hard-01",
3117
+ "tier": "hard",
3118
+ "expected": "yes",
3119
+ "predicted": "yes",
3120
+ "correct": true,
3121
+ "valid": true,
3122
+ "error": null,
3123
+ "probs": {
3124
+ "yes": 0.8119,
3125
+ "no": 0.1881
3126
+ },
3127
+ "latency_s": 0.068
3128
+ },
3129
+ {
3130
+ "id": "hard-sol-b-judge_hard-02",
3131
+ "tier": "hard",
3132
+ "expected": "no",
3133
+ "predicted": "yes",
3134
+ "correct": false,
3135
+ "valid": true,
3136
+ "error": null,
3137
+ "probs": {
3138
+ "yes": 0.8809,
3139
+ "no": 0.1191
3140
+ },
3141
+ "latency_s": 0.068
3142
+ },
3143
+ {
3144
+ "id": "hard-sol-b-judge_hard-05",
3145
+ "tier": "hard",
3146
+ "expected": "yes",
3147
+ "predicted": "yes",
3148
+ "correct": true,
3149
+ "valid": true,
3150
+ "error": null,
3151
+ "probs": {
3152
+ "yes": 0.8726,
3153
+ "no": 0.1274
3154
+ },
3155
+ "latency_s": 0.068
3156
+ },
3157
+ {
3158
+ "id": "hard-sol-b-judge_hard-08",
3159
+ "tier": "hard",
3160
+ "expected": "no",
3161
+ "predicted": "yes",
3162
+ "correct": false,
3163
+ "valid": true,
3164
+ "error": null,
3165
+ "probs": {
3166
+ "yes": 0.8925,
3167
+ "no": 0.1075
3168
+ },
3169
+ "latency_s": 0.068
3170
+ },
3171
+ {
3172
+ "id": "hard-sol-b-judge_hard-10",
3173
+ "tier": "hard",
3174
+ "expected": "no",
3175
+ "predicted": "no",
3176
+ "correct": true,
3177
+ "valid": true,
3178
+ "error": null,
3179
+ "probs": {
3180
+ "yes": 0.4808,
3181
+ "no": 0.5192
3182
+ },
3183
+ "latency_s": 0.068
3184
+ },
3185
+ {
3186
+ "id": "hard-sol-b-judge_hard-14",
3187
+ "tier": "hard",
3188
+ "expected": "no",
3189
+ "predicted": "no",
3190
+ "correct": true,
3191
+ "valid": true,
3192
+ "error": null,
3193
+ "probs": {
3194
+ "yes": 0.0568,
3195
+ "no": 0.9432
3196
+ },
3197
+ "latency_s": 0.068
3198
+ },
3199
+ {
3200
+ "id": "hard-sol-b-judge_hard-15",
3201
+ "tier": "hard",
3202
+ "expected": "yes",
3203
+ "predicted": "yes",
3204
+ "correct": true,
3205
+ "valid": true,
3206
+ "error": null,
3207
+ "probs": {
3208
+ "yes": 0.8683,
3209
+ "no": 0.1317
3210
+ },
3211
+ "latency_s": 0.069
3212
+ },
3213
+ {
3214
+ "id": "hard-sol-b-judge_hard-18",
3215
+ "tier": "hard",
3216
+ "expected": "no",
3217
+ "predicted": "yes",
3218
+ "correct": false,
3219
+ "valid": true,
3220
+ "error": null,
3221
+ "probs": {
3222
+ "yes": 0.9343,
3223
+ "no": 0.0657
3224
+ },
3225
+ "latency_s": 0.069
3226
+ },
3227
+ {
3228
+ "id": "hard-sol-b-long_policy-01",
3229
+ "tier": "hard",
3230
+ "expected": "reject_entire_claim",
3231
+ "predicted": "reject_entire_claim",
3232
+ "correct": true,
3233
+ "valid": true,
3234
+ "error": null,
3235
+ "probs": {
3236
+ "approve_all": 0.0068,
3237
+ "approve_lodging_only": 0.1693,
3238
+ "approve_meals_only": 0.0208,
3239
+ "reject_entire_claim": 0.8031
3240
+ },
3241
+ "latency_s": 0.094
3242
+ },
3243
+ {
3244
+ "id": "hard-sol-b-long_policy-02",
3245
+ "tier": "hard",
3246
+ "expected": "deny_current_export",
3247
+ "predicted": "privacy_only",
3248
+ "correct": false,
3249
+ "valid": true,
3250
+ "error": null,
3251
+ "probs": {
3252
+ "analytics_owner_only": 0.0193,
3253
+ "deny_current_export": 0.0823,
3254
+ "privacy_and_security": 0.0701,
3255
+ "privacy_only": 0.8283
3256
+ },
3257
+ "latency_s": 0.091
3258
+ },
3259
+ {
3260
+ "id": "hard-sol-b-long_policy-05",
3261
+ "tier": "hard",
3262
+ "expected": "delete_profile_omar_april_and_table",
3263
+ "predicted": "delete_nothing",
3264
+ "correct": false,
3265
+ "valid": true,
3266
+ "error": null,
3267
+ "probs": {
3268
+ "delete_everything": 0.0049,
3269
+ "delete_nothing": 0.728,
3270
+ "delete_only_profile": 0.2117,
3271
+ "delete_profile_omar_april_and_table": 0.0553
3272
+ },
3273
+ "latency_s": 0.088
3274
+ },
3275
+ {
3276
+ "id": "hard-sol-b-long_policy-06",
3277
+ "tier": "hard",
3278
+ "expected": "east_ward",
3279
+ "predicted": "river_arts",
3280
+ "correct": false,
3281
+ "valid": true,
3282
+ "error": null,
3283
+ "probs": {
3284
+ "east_ward": 0.0484,
3285
+ "harbor_neighbors": 0.0092,
3286
+ "no_award": 0.0633,
3287
+ "river_arts": 0.8791
3288
+ },
3289
+ "latency_s": 0.09
3290
+ },
3291
+ {
3292
+ "id": "hard-sol-b-routing_hard-01",
3293
+ "tier": "hard",
3294
+ "expected": "office_file_repair",
3295
+ "predicted": "office_file_repair",
3296
+ "correct": true,
3297
+ "valid": true,
3298
+ "error": null,
3299
+ "probs": {
3300
+ "data_pipeline": 0.0006,
3301
+ "desktop_support": 0.0015,
3302
+ "finance_review": 0.0017,
3303
+ "office_file_repair": 0.9875,
3304
+ "spreadsheet_formula": 0.0088
3305
+ },
3306
+ "latency_s": 0.069
3307
+ },
3308
+ {
3309
+ "id": "hard-sol-b-routing_hard-02",
3310
+ "tier": "hard",
3311
+ "expected": "ledger_reconciliation",
3312
+ "predicted": "ledger_reconciliation",
3313
+ "correct": true,
3314
+ "valid": true,
3315
+ "error": null,
3316
+ "probs": {
3317
+ "customer_support": 0.0035,
3318
+ "fraud_review": 0.0006,
3319
+ "ledger_reconciliation": 0.99,
3320
+ "mobile_app": 0.001,
3321
+ "payment_disputes": 0.0049
3322
+ },
3323
+ "latency_s": 0.069
3324
+ },
3325
+ {
3326
+ "id": "hard-sol-b-routing_hard-03",
3327
+ "tier": "hard",
3328
+ "expected": "network_dns",
3329
+ "predicted": "network_dns",
3330
+ "correct": true,
3331
+ "valid": true,
3332
+ "error": null,
3333
+ "probs": {
3334
+ "api_platform": 0.0068,
3335
+ "authoritative_dns": 0.0094,
3336
+ "endpoint_devices": 0.0276,
3337
+ "network_dns": 0.9545,
3338
+ "tls_certificates": 0.0017
3339
+ },
3340
+ "latency_s": 0.068
3341
+ },
3342
+ {
3343
+ "id": "hard-sol-b-routing_hard-07",
3344
+ "tier": "hard",
3345
+ "expected": "release_notarization",
3346
+ "predicted": "release_notarization",
3347
+ "correct": true,
3348
+ "valid": true,
3349
+ "error": null,
3350
+ "probs": {
3351
+ "ci_infrastructure": 0.001,
3352
+ "code_signing": 0.0067,
3353
+ "endpoint_security": 0.001,
3354
+ "macos_runtime": 0.0018,
3355
+ "release_notarization": 0.9895
3356
+ },
3357
+ "latency_s": 0.069
3358
+ },
3359
+ {
3360
+ "id": "hard-sol-b-routing_hard-09",
3361
+ "tier": "hard",
3362
+ "expected": "source_access",
3363
+ "predicted": "source_access",
3364
+ "correct": true,
3365
+ "valid": true,
3366
+ "error": null,
3367
+ "probs": {
3368
+ "git_client": 0.0028,
3369
+ "identity_login": 0.0013,
3370
+ "repository_admin": 0.0018,
3371
+ "source_access": 0.9918,
3372
+ "vpn_network": 0.0023
3373
+ },
3374
+ "latency_s": 0.069
3375
+ },
3376
+ {
3377
+ "id": "hard-sol-b-temporal_numeric-01",
3378
+ "tier": "hard",
3379
+ "expected": "late_by_under_2h",
3380
+ "predicted": "within_window",
3381
+ "correct": false,
3382
+ "valid": true,
3383
+ "error": null,
3384
+ "probs": {
3385
+ "before_open": 0.1334,
3386
+ "late_by_2h_or_more": 0.0823,
3387
+ "late_by_under_2h": 0.1842,
3388
+ "within_window": 0.6001
3389
+ },
3390
+ "latency_s": 0.068
3391
+ },
3392
+ {
3393
+ "id": "hard-sol-b-temporal_numeric-02",
3394
+ "tier": "hard",
3395
+ "expected": "over_limit",
3396
+ "predicted": "under_100gb_below",
3397
+ "correct": false,
3398
+ "valid": true,
3399
+ "error": null,
3400
+ "probs": {
3401
+ "at_least_100gb_below": 0.1587,
3402
+ "exactly_equal": 0.1675,
3403
+ "over_limit": 0.128,
3404
+ "under_100gb_below": 0.5457
3405
+ },
3406
+ "latency_s": 0.068
3407
+ },
3408
+ {
3409
+ "id": "hard-sol-b-temporal_numeric-03",
3410
+ "tier": "hard",
3411
+ "expected": "9_credits",
3412
+ "predicted": "9_credits",
3413
+ "correct": true,
3414
+ "valid": true,
3415
+ "error": null,
3416
+ "probs": {
3417
+ "12_credits": 0.184,
3418
+ "15_credits": 0.0596,
3419
+ "6_credits": 0.1567,
3420
+ "9_credits": 0.5997
3421
+ },
3422
+ "latency_s": 0.068
3423
+ },
3424
+ {
3425
+ "id": "hard-sol-b-temporal_numeric-04",
3426
+ "tier": "hard",
3427
+ "expected": "pass",
3428
+ "predicted": "narrow_fail",
3429
+ "correct": false,
3430
+ "valid": true,
3431
+ "error": null,
3432
+ "probs": {
3433
+ "clear_fail": 0.1773,
3434
+ "narrow_fail": 0.5779,
3435
+ "pass": 0.2448
3436
+ },
3437
+ "latency_s": 0.068
3438
+ },
3439
+ {
3440
+ "id": "hard-sol-c-judge_hard-03",
3441
+ "tier": "hard",
3442
+ "expected": "no",
3443
+ "predicted": "yes",
3444
+ "correct": false,
3445
+ "valid": true,
3446
+ "error": null,
3447
+ "probs": {
3448
+ "yes": 0.806,
3449
+ "no": 0.194
3450
+ },
3451
+ "latency_s": 0.07
3452
+ },
3453
+ {
3454
+ "id": "hard-sol-c-judge_hard-05",
3455
+ "tier": "hard",
3456
+ "expected": "yes",
3457
+ "predicted": "yes",
3458
+ "correct": true,
3459
+ "valid": true,
3460
+ "error": null,
3461
+ "probs": {
3462
+ "yes": 0.7673,
3463
+ "no": 0.2327
3464
+ },
3465
+ "latency_s": 0.069
3466
+ },
3467
+ {
3468
+ "id": "hard-sol-c-judge_hard-07",
3469
+ "tier": "hard",
3470
+ "expected": "no",
3471
+ "predicted": "yes",
3472
+ "correct": false,
3473
+ "valid": true,
3474
+ "error": null,
3475
+ "probs": {
3476
+ "yes": 0.6493,
3477
+ "no": 0.3507
3478
+ },
3479
+ "latency_s": 0.069
3480
+ },
3481
+ {
3482
+ "id": "hard-sol-c-judge_hard-08",
3483
+ "tier": "hard",
3484
+ "expected": "no",
3485
+ "predicted": "yes",
3486
+ "correct": false,
3487
+ "valid": true,
3488
+ "error": null,
3489
+ "probs": {
3490
+ "yes": 0.9158,
3491
+ "no": 0.0842
3492
+ },
3493
+ "latency_s": 0.069
3494
+ },
3495
+ {
3496
+ "id": "hard-sol-c-judge_hard-09",
3497
+ "tier": "hard",
3498
+ "expected": "yes",
3499
+ "predicted": "yes",
3500
+ "correct": true,
3501
+ "valid": true,
3502
+ "error": null,
3503
+ "probs": {
3504
+ "yes": 0.8809,
3505
+ "no": 0.1191
3506
+ },
3507
+ "latency_s": 0.069
3508
+ },
3509
+ {
3510
+ "id": "hard-sol-c-judge_hard-10",
3511
+ "tier": "hard",
3512
+ "expected": "yes",
3513
+ "predicted": "yes",
3514
+ "correct": true,
3515
+ "valid": true,
3516
+ "error": null,
3517
+ "probs": {
3518
+ "yes": 0.8289,
3519
+ "no": 0.1711
3520
+ },
3521
+ "latency_s": 0.069
3522
+ },
3523
+ {
3524
+ "id": "hard-sol-c-judge_hard-11",
3525
+ "tier": "hard",
3526
+ "expected": "no",
3527
+ "predicted": "no",
3528
+ "correct": true,
3529
+ "valid": true,
3530
+ "error": null,
3531
+ "probs": {
3532
+ "yes": 0.0936,
3533
+ "no": 0.9064
3534
+ },
3535
+ "latency_s": 0.069
3536
+ },
3537
+ {
3538
+ "id": "hard-sol-c-judge_hard-13",
3539
+ "tier": "hard",
3540
+ "expected": "no",
3541
+ "predicted": "yes",
3542
+ "correct": false,
3543
+ "valid": true,
3544
+ "error": null,
3545
+ "probs": {
3546
+ "yes": 0.6666,
3547
+ "no": 0.3334
3548
+ },
3549
+ "latency_s": 0.069
3550
+ },
3551
+ {
3552
+ "id": "hard-sol-c-judge_hard-15",
3553
+ "tier": "hard",
3554
+ "expected": "yes",
3555
+ "predicted": "yes",
3556
+ "correct": true,
3557
+ "valid": true,
3558
+ "error": null,
3559
+ "probs": {
3560
+ "yes": 0.9432,
3561
+ "no": 0.0568
3562
+ },
3563
+ "latency_s": 0.069
3564
+ },
3565
+ {
3566
+ "id": "hard-sol-c-multi_hop-07",
3567
+ "tier": "hard",
3568
+ "expected": "programme_review",
3569
+ "predicted": "release_draw",
3570
+ "correct": false,
3571
+ "valid": true,
3572
+ "error": null,
3573
+ "probs": {
3574
+ "partial_release": 0.0218,
3575
+ "programme_review": 0.287,
3576
+ "reject_draw": 0.0135,
3577
+ "release_draw": 0.6777
3578
+ },
3579
+ "latency_s": 0.087
3580
+ },
3581
+ {
3582
+ "id": "hard-sol-c-multi_hop-09",
3583
+ "tier": "hard",
3584
+ "expected": "bus_substitution",
3585
+ "predicted": "pass_fen_cross",
3586
+ "correct": false,
3587
+ "valid": true,
3588
+ "error": null,
3589
+ "probs": {
3590
+ "bus_substitution": 0.2564,
3591
+ "call_fen_cross": 0.1346,
3592
+ "pass_fen_cross": 0.5738,
3593
+ "terminate_oakmere": 0.0352
3594
+ },
3595
+ "latency_s": 0.087
3596
+ },
3597
+ {
3598
+ "id": "hard-sol-c-multi_hop-11",
3599
+ "tier": "hard",
3600
+ "expected": "coverage_excluded",
3601
+ "predicted": "coverage_excluded",
3602
+ "correct": true,
3603
+ "valid": true,
3604
+ "error": null,
3605
+ "probs": {
3606
+ "adjuster_referral": 0.0112,
3607
+ "coverage_confirmed": 0.4303,
3608
+ "coverage_excluded": 0.5055,
3609
+ "documentation_needed": 0.053
3610
+ },
3611
+ "latency_s": 0.089
3612
+ },
3613
+ {
3614
+ "id": "hard-sol-c-multi_hop-12",
3615
+ "tier": "hard",
3616
+ "expected": "conservation_hold",
3617
+ "predicted": "supervised_delivery",
3618
+ "correct": false,
3619
+ "valid": true,
3620
+ "error": null,
3621
+ "probs": {
3622
+ "conservation_hold": 0.1312,
3623
+ "deny_request": 0.1003,
3624
+ "digital_copy_only": 0.1461,
3625
+ "supervised_delivery": 0.6225
3626
+ },
3627
+ "latency_s": 0.07
3628
+ },
3629
+ {
3630
+ "id": "hard-sol-c-multi_hop-13",
3631
+ "tier": "hard",
3632
+ "expected": "normal_release",
3633
+ "predicted": "normal_release",
3634
+ "correct": true,
3635
+ "valid": true,
3636
+ "error": null,
3637
+ "probs": {
3638
+ "engineering_review": 0.0004,
3639
+ "normal_release": 0.9984,
3640
+ "rework": 0.0007,
3641
+ "scrap_lot": 0.0006
3642
+ },
3643
+ "latency_s": 0.09
3644
+ }
3645
+ ]
3646
+ }
eval/jevbench_public231_this_model.json ADDED
@@ -0,0 +1,3646 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "/content/runs/delta/model",
3
+ "summary": {
4
+ "easy": {
5
+ "n": 48,
6
+ "accuracy": 1.0,
7
+ "invalid": 0,
8
+ "p50_s": 0.067
9
+ },
10
+ "standard": {
11
+ "n": 72,
12
+ "accuracy": 0.8889,
13
+ "invalid": 0,
14
+ "p50_s": 0.068
15
+ },
16
+ "hard": {
17
+ "n": 111,
18
+ "accuracy": 0.6577,
19
+ "invalid": 0,
20
+ "p50_s": 0.071
21
+ }
22
+ },
23
+ "rows": [
24
+ {
25
+ "id": "easy-intent-00",
26
+ "tier": "easy",
27
+ "expected": "track_order",
28
+ "predicted": "track_order",
29
+ "correct": true,
30
+ "valid": true,
31
+ "error": null,
32
+ "probs": {
33
+ "billing_question": 0.0,
34
+ "cancel_order": 0.0,
35
+ "change_address": 0.0,
36
+ "report_damage": 0.0,
37
+ "track_order": 1.0
38
+ },
39
+ "latency_s": 1.954
40
+ },
41
+ {
42
+ "id": "easy-intent-01",
43
+ "tier": "easy",
44
+ "expected": "cancel_order",
45
+ "predicted": "cancel_order",
46
+ "correct": true,
47
+ "valid": true,
48
+ "error": null,
49
+ "probs": {
50
+ "billing_question": 0.0,
51
+ "cancel_order": 0.9999,
52
+ "change_address": 0.0,
53
+ "report_damage": 0.0,
54
+ "track_order": 0.0
55
+ },
56
+ "latency_s": 0.069
57
+ },
58
+ {
59
+ "id": "easy-intent-02",
60
+ "tier": "easy",
61
+ "expected": "change_address",
62
+ "predicted": "change_address",
63
+ "correct": true,
64
+ "valid": true,
65
+ "error": null,
66
+ "probs": {
67
+ "billing_question": 0.0,
68
+ "cancel_order": 0.0,
69
+ "change_address": 0.9998,
70
+ "report_damage": 0.0001,
71
+ "track_order": 0.0
72
+ },
73
+ "latency_s": 0.067
74
+ },
75
+ {
76
+ "id": "easy-intent-03",
77
+ "tier": "easy",
78
+ "expected": "report_damage",
79
+ "predicted": "report_damage",
80
+ "correct": true,
81
+ "valid": true,
82
+ "error": null,
83
+ "probs": {
84
+ "billing_question": 0.0001,
85
+ "cancel_order": 0.0004,
86
+ "change_address": 0.0001,
87
+ "report_damage": 0.9994,
88
+ "track_order": 0.0001
89
+ },
90
+ "latency_s": 0.067
91
+ },
92
+ {
93
+ "id": "easy-intent-04",
94
+ "tier": "easy",
95
+ "expected": "billing_question",
96
+ "predicted": "billing_question",
97
+ "correct": true,
98
+ "valid": true,
99
+ "error": null,
100
+ "probs": {
101
+ "billing_question": 0.9981,
102
+ "cancel_order": 0.001,
103
+ "change_address": 0.0003,
104
+ "report_damage": 0.0002,
105
+ "track_order": 0.0003
106
+ },
107
+ "latency_s": 0.067
108
+ },
109
+ {
110
+ "id": "easy-intent-05",
111
+ "tier": "easy",
112
+ "expected": "track_order",
113
+ "predicted": "track_order",
114
+ "correct": true,
115
+ "valid": true,
116
+ "error": null,
117
+ "probs": {
118
+ "billing_question": 0.0,
119
+ "cancel_order": 0.0001,
120
+ "change_address": 0.0,
121
+ "report_damage": 0.0,
122
+ "track_order": 0.9998
123
+ },
124
+ "latency_s": 0.067
125
+ },
126
+ {
127
+ "id": "easy-intent-06",
128
+ "tier": "easy",
129
+ "expected": "set_alarm",
130
+ "predicted": "set_alarm",
131
+ "correct": true,
132
+ "valid": true,
133
+ "error": null,
134
+ "probs": {
135
+ "play_music": 0.0,
136
+ "send_message": 0.0001,
137
+ "set_alarm": 0.9998,
138
+ "turn_off_lights": 0.0001,
139
+ "weather": 0.0001
140
+ },
141
+ "latency_s": 0.067
142
+ },
143
+ {
144
+ "id": "easy-intent-07",
145
+ "tier": "easy",
146
+ "expected": "play_music",
147
+ "predicted": "play_music",
148
+ "correct": true,
149
+ "valid": true,
150
+ "error": null,
151
+ "probs": {
152
+ "play_music": 0.9993,
153
+ "send_message": 0.0002,
154
+ "set_alarm": 0.0001,
155
+ "turn_off_lights": 0.0001,
156
+ "weather": 0.0002
157
+ },
158
+ "latency_s": 0.066
159
+ },
160
+ {
161
+ "id": "easy-intent-08",
162
+ "tier": "easy",
163
+ "expected": "weather",
164
+ "predicted": "weather",
165
+ "correct": true,
166
+ "valid": true,
167
+ "error": null,
168
+ "probs": {
169
+ "play_music": 0.0,
170
+ "send_message": 0.0,
171
+ "set_alarm": 0.0,
172
+ "turn_off_lights": 0.0,
173
+ "weather": 1.0
174
+ },
175
+ "latency_s": 0.066
176
+ },
177
+ {
178
+ "id": "easy-intent-09",
179
+ "tier": "easy",
180
+ "expected": "send_message",
181
+ "predicted": "send_message",
182
+ "correct": true,
183
+ "valid": true,
184
+ "error": null,
185
+ "probs": {
186
+ "play_music": 0.0002,
187
+ "send_message": 0.9992,
188
+ "set_alarm": 0.0002,
189
+ "turn_off_lights": 0.0002,
190
+ "weather": 0.0003
191
+ },
192
+ "latency_s": 0.066
193
+ },
194
+ {
195
+ "id": "easy-intent-10",
196
+ "tier": "easy",
197
+ "expected": "turn_off_lights",
198
+ "predicted": "turn_off_lights",
199
+ "correct": true,
200
+ "valid": true,
201
+ "error": null,
202
+ "probs": {
203
+ "play_music": 0.0001,
204
+ "send_message": 0.0001,
205
+ "set_alarm": 0.0001,
206
+ "turn_off_lights": 0.9995,
207
+ "weather": 0.0001
208
+ },
209
+ "latency_s": 0.067
210
+ },
211
+ {
212
+ "id": "easy-intent-11",
213
+ "tier": "easy",
214
+ "expected": "set_alarm",
215
+ "predicted": "set_alarm",
216
+ "correct": true,
217
+ "valid": true,
218
+ "error": null,
219
+ "probs": {
220
+ "play_music": 0.0,
221
+ "send_message": 0.0,
222
+ "set_alarm": 0.9998,
223
+ "turn_off_lights": 0.0001,
224
+ "weather": 0.0001
225
+ },
226
+ "latency_s": 0.067
227
+ },
228
+ {
229
+ "id": "easy-fact-00",
230
+ "tier": "easy",
231
+ "expected": "yes",
232
+ "predicted": "yes",
233
+ "correct": true,
234
+ "valid": true,
235
+ "error": null,
236
+ "probs": {
237
+ "yes": 0.989,
238
+ "no": 0.011
239
+ },
240
+ "latency_s": 0.067
241
+ },
242
+ {
243
+ "id": "easy-fact-01",
244
+ "tier": "easy",
245
+ "expected": "no",
246
+ "predicted": "no",
247
+ "correct": true,
248
+ "valid": true,
249
+ "error": null,
250
+ "probs": {
251
+ "yes": 0.0041,
252
+ "no": 0.9959
253
+ },
254
+ "latency_s": 0.068
255
+ },
256
+ {
257
+ "id": "easy-fact-02",
258
+ "tier": "easy",
259
+ "expected": "yes",
260
+ "predicted": "yes",
261
+ "correct": true,
262
+ "valid": true,
263
+ "error": null,
264
+ "probs": {
265
+ "yes": 0.9916,
266
+ "no": 0.0084
267
+ },
268
+ "latency_s": 0.068
269
+ },
270
+ {
271
+ "id": "easy-fact-03",
272
+ "tier": "easy",
273
+ "expected": "no",
274
+ "predicted": "no",
275
+ "correct": true,
276
+ "valid": true,
277
+ "error": null,
278
+ "probs": {
279
+ "yes": 0.002,
280
+ "no": 0.998
281
+ },
282
+ "latency_s": 0.067
283
+ },
284
+ {
285
+ "id": "easy-fact-04",
286
+ "tier": "easy",
287
+ "expected": "yes",
288
+ "predicted": "yes",
289
+ "correct": true,
290
+ "valid": true,
291
+ "error": null,
292
+ "probs": {
293
+ "yes": 0.9913,
294
+ "no": 0.0087
295
+ },
296
+ "latency_s": 0.066
297
+ },
298
+ {
299
+ "id": "easy-fact-05",
300
+ "tier": "easy",
301
+ "expected": "no",
302
+ "predicted": "no",
303
+ "correct": true,
304
+ "valid": true,
305
+ "error": null,
306
+ "probs": {
307
+ "yes": 0.0091,
308
+ "no": 0.9909
309
+ },
310
+ "latency_s": 0.067
311
+ },
312
+ {
313
+ "id": "easy-fact-06",
314
+ "tier": "easy",
315
+ "expected": "yes",
316
+ "predicted": "yes",
317
+ "correct": true,
318
+ "valid": true,
319
+ "error": null,
320
+ "probs": {
321
+ "yes": 0.9791,
322
+ "no": 0.0209
323
+ },
324
+ "latency_s": 0.067
325
+ },
326
+ {
327
+ "id": "easy-fact-07",
328
+ "tier": "easy",
329
+ "expected": "no",
330
+ "predicted": "no",
331
+ "correct": true,
332
+ "valid": true,
333
+ "error": null,
334
+ "probs": {
335
+ "yes": 0.0051,
336
+ "no": 0.9949
337
+ },
338
+ "latency_s": 0.067
339
+ },
340
+ {
341
+ "id": "easy-fact-08",
342
+ "tier": "easy",
343
+ "expected": "yes",
344
+ "predicted": "yes",
345
+ "correct": true,
346
+ "valid": true,
347
+ "error": null,
348
+ "probs": {
349
+ "yes": 0.9928,
350
+ "no": 0.0072
351
+ },
352
+ "latency_s": 0.074
353
+ },
354
+ {
355
+ "id": "easy-fact-09",
356
+ "tier": "easy",
357
+ "expected": "no",
358
+ "predicted": "no",
359
+ "correct": true,
360
+ "valid": true,
361
+ "error": null,
362
+ "probs": {
363
+ "yes": 0.0041,
364
+ "no": 0.9959
365
+ },
366
+ "latency_s": 0.067
367
+ },
368
+ {
369
+ "id": "easy-fact-10",
370
+ "tier": "easy",
371
+ "expected": "yes",
372
+ "predicted": "yes",
373
+ "correct": true,
374
+ "valid": true,
375
+ "error": null,
376
+ "probs": {
377
+ "yes": 0.9718,
378
+ "no": 0.0282
379
+ },
380
+ "latency_s": 0.067
381
+ },
382
+ {
383
+ "id": "easy-fact-11",
384
+ "tier": "easy",
385
+ "expected": "no",
386
+ "predicted": "no",
387
+ "correct": true,
388
+ "valid": true,
389
+ "error": null,
390
+ "probs": {
391
+ "yes": 0.0094,
392
+ "no": 0.9906
393
+ },
394
+ "latency_s": 0.067
395
+ },
396
+ {
397
+ "id": "easy-extraction-00",
398
+ "tier": "easy",
399
+ "expected": "paypal",
400
+ "predicted": "paypal",
401
+ "correct": true,
402
+ "valid": true,
403
+ "error": null,
404
+ "probs": {
405
+ "bank_transfer": 0.0001,
406
+ "cash": 0.0001,
407
+ "credit_card": 0.0001,
408
+ "paypal": 0.9997
409
+ },
410
+ "latency_s": 0.068
411
+ },
412
+ {
413
+ "id": "easy-extraction-01",
414
+ "tier": "easy",
415
+ "expected": "cash",
416
+ "predicted": "cash",
417
+ "correct": true,
418
+ "valid": true,
419
+ "error": null,
420
+ "probs": {
421
+ "bank_transfer": 0.0001,
422
+ "cash": 0.9995,
423
+ "credit_card": 0.0003,
424
+ "paypal": 0.0001
425
+ },
426
+ "latency_s": 0.067
427
+ },
428
+ {
429
+ "id": "easy-extraction-02",
430
+ "tier": "easy",
431
+ "expected": "bank_transfer",
432
+ "predicted": "bank_transfer",
433
+ "correct": true,
434
+ "valid": true,
435
+ "error": null,
436
+ "probs": {
437
+ "bank_transfer": 0.9997,
438
+ "cash": 0.0001,
439
+ "credit_card": 0.0001,
440
+ "paypal": 0.0001
441
+ },
442
+ "latency_s": 0.067
443
+ },
444
+ {
445
+ "id": "easy-extraction-03",
446
+ "tier": "easy",
447
+ "expected": "high",
448
+ "predicted": "high",
449
+ "correct": true,
450
+ "valid": true,
451
+ "error": null,
452
+ "probs": {
453
+ "high": 0.9833,
454
+ "low": 0.0012,
455
+ "medium": 0.0013,
456
+ "urgent": 0.0141
457
+ },
458
+ "latency_s": 0.067
459
+ },
460
+ {
461
+ "id": "easy-extraction-04",
462
+ "tier": "easy",
463
+ "expected": "low",
464
+ "predicted": "low",
465
+ "correct": true,
466
+ "valid": true,
467
+ "error": null,
468
+ "probs": {
469
+ "high": 0.0009,
470
+ "low": 0.9981,
471
+ "medium": 0.0008,
472
+ "urgent": 0.0002
473
+ },
474
+ "latency_s": 0.068
475
+ },
476
+ {
477
+ "id": "easy-extraction-05",
478
+ "tier": "easy",
479
+ "expected": "urgent",
480
+ "predicted": "urgent",
481
+ "correct": true,
482
+ "valid": true,
483
+ "error": null,
484
+ "probs": {
485
+ "high": 0.0009,
486
+ "low": 0.0002,
487
+ "medium": 0.0003,
488
+ "urgent": 0.9987
489
+ },
490
+ "latency_s": 0.068
491
+ },
492
+ {
493
+ "id": "easy-extraction-06",
494
+ "tier": "easy",
495
+ "expected": "M",
496
+ "predicted": "M",
497
+ "correct": true,
498
+ "valid": true,
499
+ "error": null,
500
+ "probs": {
501
+ "L": 0.0007,
502
+ "M": 0.9989,
503
+ "S": 0.0002,
504
+ "XL": 0.0002
505
+ },
506
+ "latency_s": 0.067
507
+ },
508
+ {
509
+ "id": "easy-extraction-07",
510
+ "tier": "easy",
511
+ "expected": "XL",
512
+ "predicted": "XL",
513
+ "correct": true,
514
+ "valid": true,
515
+ "error": null,
516
+ "probs": {
517
+ "L": 0.0004,
518
+ "M": 0.0001,
519
+ "S": 0.0001,
520
+ "XL": 0.9994
521
+ },
522
+ "latency_s": 0.067
523
+ },
524
+ {
525
+ "id": "easy-extraction-08",
526
+ "tier": "easy",
527
+ "expected": "thursday",
528
+ "predicted": "thursday",
529
+ "correct": true,
530
+ "valid": true,
531
+ "error": null,
532
+ "probs": {
533
+ "friday": 0.0007,
534
+ "monday": 0.0004,
535
+ "thursday": 0.9983,
536
+ "tuesday": 0.0003,
537
+ "wednesday": 0.0003
538
+ },
539
+ "latency_s": 0.067
540
+ },
541
+ {
542
+ "id": "easy-extraction-09",
543
+ "tier": "easy",
544
+ "expected": "monday",
545
+ "predicted": "monday",
546
+ "correct": true,
547
+ "valid": true,
548
+ "error": null,
549
+ "probs": {
550
+ "friday": 0.0092,
551
+ "monday": 0.9815,
552
+ "thursday": 0.0033,
553
+ "tuesday": 0.0031,
554
+ "wednesday": 0.0028
555
+ },
556
+ "latency_s": 0.067
557
+ },
558
+ {
559
+ "id": "easy-extraction-10",
560
+ "tier": "easy",
561
+ "expected": "friday",
562
+ "predicted": "friday",
563
+ "correct": true,
564
+ "valid": true,
565
+ "error": null,
566
+ "probs": {
567
+ "friday": 0.9972,
568
+ "monday": 0.0008,
569
+ "thursday": 0.0007,
570
+ "tuesday": 0.0005,
571
+ "wednesday": 0.0007
572
+ },
573
+ "latency_s": 0.065
574
+ },
575
+ {
576
+ "id": "easy-extraction-11",
577
+ "tier": "easy",
578
+ "expected": "credit_card",
579
+ "predicted": "credit_card",
580
+ "correct": true,
581
+ "valid": true,
582
+ "error": null,
583
+ "probs": {
584
+ "bank_transfer": 0.0001,
585
+ "cash": 0.0001,
586
+ "credit_card": 0.9997,
587
+ "paypal": 0.0001
588
+ },
589
+ "latency_s": 0.068
590
+ },
591
+ {
592
+ "id": "easy-tool_selection-00",
593
+ "tier": "easy",
594
+ "expected": "get_weather",
595
+ "predicted": "get_weather",
596
+ "correct": true,
597
+ "valid": true,
598
+ "error": null,
599
+ "probs": {
600
+ "convert_currency": 0.0,
601
+ "create_calendar_event": 0.0,
602
+ "get_weather": 0.9998,
603
+ "send_email": 0.0,
604
+ "translate_text": 0.0001
605
+ },
606
+ "latency_s": 0.067
607
+ },
608
+ {
609
+ "id": "easy-tool_selection-01",
610
+ "tier": "easy",
611
+ "expected": "send_email",
612
+ "predicted": "send_email",
613
+ "correct": true,
614
+ "valid": true,
615
+ "error": null,
616
+ "probs": {
617
+ "convert_currency": 0.0001,
618
+ "create_calendar_event": 0.0001,
619
+ "get_weather": 0.0001,
620
+ "send_email": 0.9997,
621
+ "translate_text": 0.0001
622
+ },
623
+ "latency_s": 0.067
624
+ },
625
+ {
626
+ "id": "easy-tool_selection-02",
627
+ "tier": "easy",
628
+ "expected": "create_calendar_event",
629
+ "predicted": "create_calendar_event",
630
+ "correct": true,
631
+ "valid": true,
632
+ "error": null,
633
+ "probs": {
634
+ "convert_currency": 0.0,
635
+ "create_calendar_event": 0.9998,
636
+ "get_weather": 0.0001,
637
+ "send_email": 0.0,
638
+ "translate_text": 0.0001
639
+ },
640
+ "latency_s": 0.067
641
+ },
642
+ {
643
+ "id": "easy-tool_selection-03",
644
+ "tier": "easy",
645
+ "expected": "convert_currency",
646
+ "predicted": "convert_currency",
647
+ "correct": true,
648
+ "valid": true,
649
+ "error": null,
650
+ "probs": {
651
+ "convert_currency": 0.9997,
652
+ "create_calendar_event": 0.0001,
653
+ "get_weather": 0.0001,
654
+ "send_email": 0.0001,
655
+ "translate_text": 0.0001
656
+ },
657
+ "latency_s": 0.067
658
+ },
659
+ {
660
+ "id": "easy-tool_selection-04",
661
+ "tier": "easy",
662
+ "expected": "translate_text",
663
+ "predicted": "translate_text",
664
+ "correct": true,
665
+ "valid": true,
666
+ "error": null,
667
+ "probs": {
668
+ "convert_currency": 0.0,
669
+ "create_calendar_event": 0.0,
670
+ "get_weather": 0.0,
671
+ "send_email": 0.0,
672
+ "translate_text": 0.9999
673
+ },
674
+ "latency_s": 0.067
675
+ },
676
+ {
677
+ "id": "easy-tool_selection-05",
678
+ "tier": "easy",
679
+ "expected": "create_calendar_event",
680
+ "predicted": "create_calendar_event",
681
+ "correct": true,
682
+ "valid": true,
683
+ "error": null,
684
+ "probs": {
685
+ "convert_currency": 0.0,
686
+ "create_calendar_event": 0.9997,
687
+ "get_weather": 0.0001,
688
+ "send_email": 0.0001,
689
+ "translate_text": 0.0001
690
+ },
691
+ "latency_s": 0.067
692
+ },
693
+ {
694
+ "id": "easy-tool_selection-06",
695
+ "tier": "easy",
696
+ "expected": "search_flights",
697
+ "predicted": "search_flights",
698
+ "correct": true,
699
+ "valid": true,
700
+ "error": null,
701
+ "probs": {
702
+ "book_restaurant": 0.0,
703
+ "call_taxi": 0.0,
704
+ "get_stock_price": 0.0,
705
+ "search_flights": 0.9999,
706
+ "set_reminder": 0.0001
707
+ },
708
+ "latency_s": 0.067
709
+ },
710
+ {
711
+ "id": "easy-tool_selection-07",
712
+ "tier": "easy",
713
+ "expected": "book_restaurant",
714
+ "predicted": "book_restaurant",
715
+ "correct": true,
716
+ "valid": true,
717
+ "error": null,
718
+ "probs": {
719
+ "book_restaurant": 0.9993,
720
+ "call_taxi": 0.0001,
721
+ "get_stock_price": 0.0001,
722
+ "search_flights": 0.0001,
723
+ "set_reminder": 0.0004
724
+ },
725
+ "latency_s": 0.068
726
+ },
727
+ {
728
+ "id": "easy-tool_selection-08",
729
+ "tier": "easy",
730
+ "expected": "set_reminder",
731
+ "predicted": "set_reminder",
732
+ "correct": true,
733
+ "valid": true,
734
+ "error": null,
735
+ "probs": {
736
+ "book_restaurant": 0.0,
737
+ "call_taxi": 0.0,
738
+ "get_stock_price": 0.0,
739
+ "search_flights": 0.0,
740
+ "set_reminder": 0.9999
741
+ },
742
+ "latency_s": 0.067
743
+ },
744
+ {
745
+ "id": "easy-tool_selection-09",
746
+ "tier": "easy",
747
+ "expected": "get_stock_price",
748
+ "predicted": "get_stock_price",
749
+ "correct": true,
750
+ "valid": true,
751
+ "error": null,
752
+ "probs": {
753
+ "book_restaurant": 0.0,
754
+ "call_taxi": 0.0,
755
+ "get_stock_price": 0.9998,
756
+ "search_flights": 0.0,
757
+ "set_reminder": 0.0001
758
+ },
759
+ "latency_s": 0.067
760
+ },
761
+ {
762
+ "id": "easy-tool_selection-10",
763
+ "tier": "easy",
764
+ "expected": "call_taxi",
765
+ "predicted": "call_taxi",
766
+ "correct": true,
767
+ "valid": true,
768
+ "error": null,
769
+ "probs": {
770
+ "book_restaurant": 0.0001,
771
+ "call_taxi": 0.9994,
772
+ "get_stock_price": 0.0001,
773
+ "search_flights": 0.0001,
774
+ "set_reminder": 0.0002
775
+ },
776
+ "latency_s": 0.067
777
+ },
778
+ {
779
+ "id": "easy-tool_selection-11",
780
+ "tier": "easy",
781
+ "expected": "set_reminder",
782
+ "predicted": "set_reminder",
783
+ "correct": true,
784
+ "valid": true,
785
+ "error": null,
786
+ "probs": {
787
+ "book_restaurant": 0.0,
788
+ "call_taxi": 0.0,
789
+ "get_stock_price": 0.0,
790
+ "search_flights": 0.0,
791
+ "set_reminder": 0.9999
792
+ },
793
+ "latency_s": 0.066
794
+ },
795
+ {
796
+ "id": "original-policy-01-0",
797
+ "tier": "standard",
798
+ "expected": "no",
799
+ "predicted": "no",
800
+ "correct": true,
801
+ "valid": true,
802
+ "error": null,
803
+ "probs": {
804
+ "yes": 0.0078,
805
+ "no": 0.9922
806
+ },
807
+ "latency_s": 0.067
808
+ },
809
+ {
810
+ "id": "original-policy-01-1",
811
+ "tier": "standard",
812
+ "expected": "no",
813
+ "predicted": "no",
814
+ "correct": true,
815
+ "valid": true,
816
+ "error": null,
817
+ "probs": {
818
+ "yes": 0.0118,
819
+ "no": 0.9882
820
+ },
821
+ "latency_s": 0.067
822
+ },
823
+ {
824
+ "id": "original-policy-02-0",
825
+ "tier": "standard",
826
+ "expected": "yes",
827
+ "predicted": "yes",
828
+ "correct": true,
829
+ "valid": true,
830
+ "error": null,
831
+ "probs": {
832
+ "yes": 0.9432,
833
+ "no": 0.0568
834
+ },
835
+ "latency_s": 0.067
836
+ },
837
+ {
838
+ "id": "original-policy-02-1",
839
+ "tier": "standard",
840
+ "expected": "yes",
841
+ "predicted": "no",
842
+ "correct": false,
843
+ "valid": true,
844
+ "error": null,
845
+ "probs": {
846
+ "yes": 0.194,
847
+ "no": 0.806
848
+ },
849
+ "latency_s": 0.066
850
+ },
851
+ {
852
+ "id": "original-policy-03-0",
853
+ "tier": "standard",
854
+ "expected": "yes",
855
+ "predicted": "yes",
856
+ "correct": true,
857
+ "valid": true,
858
+ "error": null,
859
+ "probs": {
860
+ "yes": 0.6134,
861
+ "no": 0.3866
862
+ },
863
+ "latency_s": 0.067
864
+ },
865
+ {
866
+ "id": "original-policy-03-1",
867
+ "tier": "standard",
868
+ "expected": "yes",
869
+ "predicted": "no",
870
+ "correct": false,
871
+ "valid": true,
872
+ "error": null,
873
+ "probs": {
874
+ "yes": 0.3249,
875
+ "no": 0.6751
876
+ },
877
+ "latency_s": 0.071
878
+ },
879
+ {
880
+ "id": "original-policy-04-0",
881
+ "tier": "standard",
882
+ "expected": "no",
883
+ "predicted": "no",
884
+ "correct": true,
885
+ "valid": true,
886
+ "error": null,
887
+ "probs": {
888
+ "yes": 0.0409,
889
+ "no": 0.9591
890
+ },
891
+ "latency_s": 0.069
892
+ },
893
+ {
894
+ "id": "original-policy-04-1",
895
+ "tier": "standard",
896
+ "expected": "no",
897
+ "predicted": "no",
898
+ "correct": true,
899
+ "valid": true,
900
+ "error": null,
901
+ "probs": {
902
+ "yes": 0.0091,
903
+ "no": 0.9909
904
+ },
905
+ "latency_s": 0.068
906
+ },
907
+ {
908
+ "id": "original-policy-05-0",
909
+ "tier": "standard",
910
+ "expected": "no",
911
+ "predicted": "no",
912
+ "correct": true,
913
+ "valid": true,
914
+ "error": null,
915
+ "probs": {
916
+ "yes": 0.0657,
917
+ "no": 0.9343
918
+ },
919
+ "latency_s": 0.068
920
+ },
921
+ {
922
+ "id": "original-policy-05-1",
923
+ "tier": "standard",
924
+ "expected": "no",
925
+ "predicted": "no",
926
+ "correct": true,
927
+ "valid": true,
928
+ "error": null,
929
+ "probs": {
930
+ "yes": 0.0611,
931
+ "no": 0.9389
932
+ },
933
+ "latency_s": 0.069
934
+ },
935
+ {
936
+ "id": "original-policy-06-0",
937
+ "tier": "standard",
938
+ "expected": "yes",
939
+ "predicted": "yes",
940
+ "correct": true,
941
+ "valid": true,
942
+ "error": null,
943
+ "probs": {
944
+ "yes": 0.9367,
945
+ "no": 0.0633
946
+ },
947
+ "latency_s": 0.07
948
+ },
949
+ {
950
+ "id": "original-policy-06-1",
951
+ "tier": "standard",
952
+ "expected": "yes",
953
+ "predicted": "yes",
954
+ "correct": true,
955
+ "valid": true,
956
+ "error": null,
957
+ "probs": {
958
+ "yes": 0.8177,
959
+ "no": 0.1823
960
+ },
961
+ "latency_s": 0.07
962
+ },
963
+ {
964
+ "id": "original-intent-01-0",
965
+ "tier": "standard",
966
+ "expected": "refund",
967
+ "predicted": "refund",
968
+ "correct": true,
969
+ "valid": true,
970
+ "error": null,
971
+ "probs": {
972
+ "cancel": 0.0027,
973
+ "change_address": 0.0003,
974
+ "other": 0.0035,
975
+ "refund": 0.9932,
976
+ "status": 0.0002
977
+ },
978
+ "latency_s": 0.07
979
+ },
980
+ {
981
+ "id": "original-intent-01-1",
982
+ "tier": "standard",
983
+ "expected": "refund",
984
+ "predicted": "refund",
985
+ "correct": true,
986
+ "valid": true,
987
+ "error": null,
988
+ "probs": {
989
+ "cancel": 0.0238,
990
+ "change_address": 0.0009,
991
+ "other": 0.0086,
992
+ "refund": 0.9663,
993
+ "status": 0.0005
994
+ },
995
+ "latency_s": 0.069
996
+ },
997
+ {
998
+ "id": "original-intent-02-0",
999
+ "tier": "standard",
1000
+ "expected": "status",
1001
+ "predicted": "status",
1002
+ "correct": true,
1003
+ "valid": true,
1004
+ "error": null,
1005
+ "probs": {
1006
+ "cancel": 0.0007,
1007
+ "change_address": 0.0023,
1008
+ "other": 0.1811,
1009
+ "refund": 0.0014,
1010
+ "status": 0.8144
1011
+ },
1012
+ "latency_s": 0.067
1013
+ },
1014
+ {
1015
+ "id": "original-intent-02-1",
1016
+ "tier": "standard",
1017
+ "expected": "status",
1018
+ "predicted": "status",
1019
+ "correct": true,
1020
+ "valid": true,
1021
+ "error": null,
1022
+ "probs": {
1023
+ "cancel": 0.0,
1024
+ "change_address": 0.0001,
1025
+ "other": 0.001,
1026
+ "refund": 0.0,
1027
+ "status": 0.9988
1028
+ },
1029
+ "latency_s": 0.067
1030
+ },
1031
+ {
1032
+ "id": "original-intent-03-0",
1033
+ "tier": "standard",
1034
+ "expected": "change_address",
1035
+ "predicted": "change_address",
1036
+ "correct": true,
1037
+ "valid": true,
1038
+ "error": null,
1039
+ "probs": {
1040
+ "cancel": 0.0007,
1041
+ "change_address": 0.9789,
1042
+ "other": 0.0184,
1043
+ "refund": 0.0011,
1044
+ "status": 0.0008
1045
+ },
1046
+ "latency_s": 0.067
1047
+ },
1048
+ {
1049
+ "id": "original-intent-03-1",
1050
+ "tier": "standard",
1051
+ "expected": "change_address",
1052
+ "predicted": "change_address",
1053
+ "correct": true,
1054
+ "valid": true,
1055
+ "error": null,
1056
+ "probs": {
1057
+ "cancel": 0.0001,
1058
+ "change_address": 0.9992,
1059
+ "other": 0.0006,
1060
+ "refund": 0.0001,
1061
+ "status": 0.0001
1062
+ },
1063
+ "latency_s": 0.067
1064
+ },
1065
+ {
1066
+ "id": "original-intent-04-0",
1067
+ "tier": "standard",
1068
+ "expected": "cancel",
1069
+ "predicted": "cancel",
1070
+ "correct": true,
1071
+ "valid": true,
1072
+ "error": null,
1073
+ "probs": {
1074
+ "cancel": 0.996,
1075
+ "change_address": 0.0005,
1076
+ "other": 0.0026,
1077
+ "refund": 0.0008,
1078
+ "status": 0.0002
1079
+ },
1080
+ "latency_s": 0.067
1081
+ },
1082
+ {
1083
+ "id": "original-intent-04-1",
1084
+ "tier": "standard",
1085
+ "expected": "cancel",
1086
+ "predicted": "cancel",
1087
+ "correct": true,
1088
+ "valid": true,
1089
+ "error": null,
1090
+ "probs": {
1091
+ "cancel": 0.9971,
1092
+ "change_address": 0.0003,
1093
+ "other": 0.0021,
1094
+ "refund": 0.0003,
1095
+ "status": 0.0002
1096
+ },
1097
+ "latency_s": 0.067
1098
+ },
1099
+ {
1100
+ "id": "original-intent-05-0",
1101
+ "tier": "standard",
1102
+ "expected": "other",
1103
+ "predicted": "other",
1104
+ "correct": true,
1105
+ "valid": true,
1106
+ "error": null,
1107
+ "probs": {
1108
+ "cancel": 0.0018,
1109
+ "change_address": 0.0017,
1110
+ "other": 0.8809,
1111
+ "refund": 0.1145,
1112
+ "status": 0.0011
1113
+ },
1114
+ "latency_s": 0.07
1115
+ },
1116
+ {
1117
+ "id": "original-intent-05-1",
1118
+ "tier": "standard",
1119
+ "expected": "other",
1120
+ "predicted": "other",
1121
+ "correct": true,
1122
+ "valid": true,
1123
+ "error": null,
1124
+ "probs": {
1125
+ "cancel": 0.0044,
1126
+ "change_address": 0.002,
1127
+ "other": 0.7689,
1128
+ "refund": 0.2236,
1129
+ "status": 0.001
1130
+ },
1131
+ "latency_s": 0.07
1132
+ },
1133
+ {
1134
+ "id": "original-intent-06-0",
1135
+ "tier": "standard",
1136
+ "expected": "status",
1137
+ "predicted": "status",
1138
+ "correct": true,
1139
+ "valid": true,
1140
+ "error": null,
1141
+ "probs": {
1142
+ "cancel": 0.0001,
1143
+ "change_address": 0.0001,
1144
+ "other": 0.0018,
1145
+ "refund": 0.0001,
1146
+ "status": 0.9979
1147
+ },
1148
+ "latency_s": 0.07
1149
+ },
1150
+ {
1151
+ "id": "original-intent-06-1",
1152
+ "tier": "standard",
1153
+ "expected": "status",
1154
+ "predicted": "status",
1155
+ "correct": true,
1156
+ "valid": true,
1157
+ "error": null,
1158
+ "probs": {
1159
+ "cancel": 0.0,
1160
+ "change_address": 0.0,
1161
+ "other": 0.0007,
1162
+ "refund": 0.0,
1163
+ "status": 0.9992
1164
+ },
1165
+ "latency_s": 0.07
1166
+ },
1167
+ {
1168
+ "id": "original-ordinal-01-0",
1169
+ "tier": "standard",
1170
+ "expected": "0",
1171
+ "predicted": "0",
1172
+ "correct": true,
1173
+ "valid": true,
1174
+ "error": null,
1175
+ "probs": {
1176
+ "0": 0.9488,
1177
+ "1": 0.0488,
1178
+ "2": 0.0015,
1179
+ "3": 0.0009
1180
+ },
1181
+ "latency_s": 0.074
1182
+ },
1183
+ {
1184
+ "id": "original-ordinal-01-1",
1185
+ "tier": "standard",
1186
+ "expected": "0",
1187
+ "predicted": "0",
1188
+ "correct": true,
1189
+ "valid": true,
1190
+ "error": null,
1191
+ "probs": {
1192
+ "0": 0.9047,
1193
+ "1": 0.0924,
1194
+ "2": 0.0021,
1195
+ "3": 0.0007
1196
+ },
1197
+ "latency_s": 0.07
1198
+ },
1199
+ {
1200
+ "id": "original-ordinal-02-0",
1201
+ "tier": "standard",
1202
+ "expected": "1",
1203
+ "predicted": "1",
1204
+ "correct": true,
1205
+ "valid": true,
1206
+ "error": null,
1207
+ "probs": {
1208
+ "0": 0.0207,
1209
+ "1": 0.9214,
1210
+ "2": 0.0553,
1211
+ "3": 0.0025
1212
+ },
1213
+ "latency_s": 0.07
1214
+ },
1215
+ {
1216
+ "id": "original-ordinal-02-1",
1217
+ "tier": "standard",
1218
+ "expected": "1",
1219
+ "predicted": "1",
1220
+ "correct": true,
1221
+ "valid": true,
1222
+ "error": null,
1223
+ "probs": {
1224
+ "0": 0.0159,
1225
+ "1": 0.9777,
1226
+ "2": 0.0045,
1227
+ "3": 0.0019
1228
+ },
1229
+ "latency_s": 0.071
1230
+ },
1231
+ {
1232
+ "id": "original-ordinal-03-0",
1233
+ "tier": "standard",
1234
+ "expected": "2",
1235
+ "predicted": "2",
1236
+ "correct": true,
1237
+ "valid": true,
1238
+ "error": null,
1239
+ "probs": {
1240
+ "0": 0.0275,
1241
+ "1": 0.04,
1242
+ "2": 0.9318,
1243
+ "3": 0.0008
1244
+ },
1245
+ "latency_s": 0.071
1246
+ },
1247
+ {
1248
+ "id": "original-ordinal-03-1",
1249
+ "tier": "standard",
1250
+ "expected": "2",
1251
+ "predicted": "2",
1252
+ "correct": true,
1253
+ "valid": true,
1254
+ "error": null,
1255
+ "probs": {
1256
+ "0": 0.0152,
1257
+ "1": 0.0527,
1258
+ "2": 0.9302,
1259
+ "3": 0.002
1260
+ },
1261
+ "latency_s": 0.07
1262
+ },
1263
+ {
1264
+ "id": "original-ordinal-04-0",
1265
+ "tier": "standard",
1266
+ "expected": "3",
1267
+ "predicted": "3",
1268
+ "correct": true,
1269
+ "valid": true,
1270
+ "error": null,
1271
+ "probs": {
1272
+ "0": 0.0041,
1273
+ "1": 0.0025,
1274
+ "2": 0.0088,
1275
+ "3": 0.9847
1276
+ },
1277
+ "latency_s": 0.07
1278
+ },
1279
+ {
1280
+ "id": "original-ordinal-04-1",
1281
+ "tier": "standard",
1282
+ "expected": "3",
1283
+ "predicted": "3",
1284
+ "correct": true,
1285
+ "valid": true,
1286
+ "error": null,
1287
+ "probs": {
1288
+ "0": 0.0074,
1289
+ "1": 0.0067,
1290
+ "2": 0.0666,
1291
+ "3": 0.9193
1292
+ },
1293
+ "latency_s": 0.069
1294
+ },
1295
+ {
1296
+ "id": "original-ordinal-05-0",
1297
+ "tier": "standard",
1298
+ "expected": "0",
1299
+ "predicted": "0",
1300
+ "correct": true,
1301
+ "valid": true,
1302
+ "error": null,
1303
+ "probs": {
1304
+ "0": 0.9809,
1305
+ "1": 0.0103,
1306
+ "2": 0.0074,
1307
+ "3": 0.0014
1308
+ },
1309
+ "latency_s": 0.068
1310
+ },
1311
+ {
1312
+ "id": "original-ordinal-05-1",
1313
+ "tier": "standard",
1314
+ "expected": "0",
1315
+ "predicted": "0",
1316
+ "correct": true,
1317
+ "valid": true,
1318
+ "error": null,
1319
+ "probs": {
1320
+ "0": 0.8155,
1321
+ "1": 0.0614,
1322
+ "2": 0.1069,
1323
+ "3": 0.0162
1324
+ },
1325
+ "latency_s": 0.068
1326
+ },
1327
+ {
1328
+ "id": "original-ordinal-06-0",
1329
+ "tier": "standard",
1330
+ "expected": "2",
1331
+ "predicted": "2",
1332
+ "correct": true,
1333
+ "valid": true,
1334
+ "error": null,
1335
+ "probs": {
1336
+ "0": 0.0286,
1337
+ "1": 0.0666,
1338
+ "2": 0.9045,
1339
+ "3": 0.0003
1340
+ },
1341
+ "latency_s": 0.068
1342
+ },
1343
+ {
1344
+ "id": "original-ordinal-06-1",
1345
+ "tier": "standard",
1346
+ "expected": "2",
1347
+ "predicted": "2",
1348
+ "correct": true,
1349
+ "valid": true,
1350
+ "error": null,
1351
+ "probs": {
1352
+ "0": 0.0052,
1353
+ "1": 0.0771,
1354
+ "2": 0.9173,
1355
+ "3": 0.0005
1356
+ },
1357
+ "latency_s": 0.068
1358
+ },
1359
+ {
1360
+ "id": "original-extraction-01-0",
1361
+ "tier": "standard",
1362
+ "expected": "pickup",
1363
+ "predicted": "pickup",
1364
+ "correct": true,
1365
+ "valid": true,
1366
+ "error": null,
1367
+ "probs": {
1368
+ "courier": 0.0027,
1369
+ "pickup": 0.9807,
1370
+ "post": 0.0009,
1371
+ "unknown": 0.0157
1372
+ },
1373
+ "latency_s": 0.067
1374
+ },
1375
+ {
1376
+ "id": "original-extraction-01-1",
1377
+ "tier": "standard",
1378
+ "expected": "pickup",
1379
+ "predicted": "pickup",
1380
+ "correct": true,
1381
+ "valid": true,
1382
+ "error": null,
1383
+ "probs": {
1384
+ "courier": 0.001,
1385
+ "pickup": 0.9867,
1386
+ "post": 0.0009,
1387
+ "unknown": 0.0114
1388
+ },
1389
+ "latency_s": 0.068
1390
+ },
1391
+ {
1392
+ "id": "original-extraction-02-0",
1393
+ "tier": "standard",
1394
+ "expected": "unknown",
1395
+ "predicted": "unknown",
1396
+ "correct": true,
1397
+ "valid": true,
1398
+ "error": null,
1399
+ "probs": {
1400
+ "courier": 0.0117,
1401
+ "pickup": 0.0028,
1402
+ "post": 0.213,
1403
+ "unknown": 0.7726
1404
+ },
1405
+ "latency_s": 0.067
1406
+ },
1407
+ {
1408
+ "id": "original-extraction-02-1",
1409
+ "tier": "standard",
1410
+ "expected": "unknown",
1411
+ "predicted": "unknown",
1412
+ "correct": true,
1413
+ "valid": true,
1414
+ "error": null,
1415
+ "probs": {
1416
+ "courier": 0.0018,
1417
+ "pickup": 0.0006,
1418
+ "post": 0.0141,
1419
+ "unknown": 0.9835
1420
+ },
1421
+ "latency_s": 0.074
1422
+ },
1423
+ {
1424
+ "id": "original-extraction-03-0",
1425
+ "tier": "standard",
1426
+ "expected": "courier",
1427
+ "predicted": "courier",
1428
+ "correct": true,
1429
+ "valid": true,
1430
+ "error": null,
1431
+ "probs": {
1432
+ "courier": 0.8907,
1433
+ "pickup": 0.0208,
1434
+ "post": 0.0046,
1435
+ "unknown": 0.0839
1436
+ },
1437
+ "latency_s": 0.078
1438
+ },
1439
+ {
1440
+ "id": "original-extraction-03-1",
1441
+ "tier": "standard",
1442
+ "expected": "courier",
1443
+ "predicted": "courier",
1444
+ "correct": true,
1445
+ "valid": true,
1446
+ "error": null,
1447
+ "probs": {
1448
+ "courier": 0.9993,
1449
+ "pickup": 0.0002,
1450
+ "post": 0.0002,
1451
+ "unknown": 0.0003
1452
+ },
1453
+ "latency_s": 0.07
1454
+ },
1455
+ {
1456
+ "id": "original-extraction-04-0",
1457
+ "tier": "standard",
1458
+ "expected": "post",
1459
+ "predicted": "post",
1460
+ "correct": true,
1461
+ "valid": true,
1462
+ "error": null,
1463
+ "probs": {
1464
+ "courier": 0.0002,
1465
+ "pickup": 0.0002,
1466
+ "post": 0.9983,
1467
+ "unknown": 0.0012
1468
+ },
1469
+ "latency_s": 0.067
1470
+ },
1471
+ {
1472
+ "id": "original-extraction-04-1",
1473
+ "tier": "standard",
1474
+ "expected": "post",
1475
+ "predicted": "post",
1476
+ "correct": true,
1477
+ "valid": true,
1478
+ "error": null,
1479
+ "probs": {
1480
+ "courier": 0.0005,
1481
+ "pickup": 0.0003,
1482
+ "post": 0.998,
1483
+ "unknown": 0.0012
1484
+ },
1485
+ "latency_s": 0.067
1486
+ },
1487
+ {
1488
+ "id": "original-extraction-05-0",
1489
+ "tier": "standard",
1490
+ "expected": "unknown",
1491
+ "predicted": "unknown",
1492
+ "correct": true,
1493
+ "valid": true,
1494
+ "error": null,
1495
+ "probs": {
1496
+ "courier": 0.0006,
1497
+ "pickup": 0.0005,
1498
+ "post": 0.0003,
1499
+ "unknown": 0.9986
1500
+ },
1501
+ "latency_s": 0.066
1502
+ },
1503
+ {
1504
+ "id": "original-extraction-05-1",
1505
+ "tier": "standard",
1506
+ "expected": "unknown",
1507
+ "predicted": "unknown",
1508
+ "correct": true,
1509
+ "valid": true,
1510
+ "error": null,
1511
+ "probs": {
1512
+ "courier": 0.4175,
1513
+ "pickup": 0.0032,
1514
+ "post": 0.0032,
1515
+ "unknown": 0.5762
1516
+ },
1517
+ "latency_s": 0.066
1518
+ },
1519
+ {
1520
+ "id": "original-extraction-06-0",
1521
+ "tier": "standard",
1522
+ "expected": "courier",
1523
+ "predicted": "courier",
1524
+ "correct": true,
1525
+ "valid": true,
1526
+ "error": null,
1527
+ "probs": {
1528
+ "courier": 0.9981,
1529
+ "pickup": 0.0004,
1530
+ "post": 0.0004,
1531
+ "unknown": 0.0011
1532
+ },
1533
+ "latency_s": 0.066
1534
+ },
1535
+ {
1536
+ "id": "original-extraction-06-1",
1537
+ "tier": "standard",
1538
+ "expected": "courier",
1539
+ "predicted": "courier",
1540
+ "correct": true,
1541
+ "valid": true,
1542
+ "error": null,
1543
+ "probs": {
1544
+ "courier": 0.9983,
1545
+ "pickup": 0.0003,
1546
+ "post": 0.0003,
1547
+ "unknown": 0.0012
1548
+ },
1549
+ "latency_s": 0.066
1550
+ },
1551
+ {
1552
+ "id": "original-adequacy-01-0",
1553
+ "tier": "standard",
1554
+ "expected": "yes",
1555
+ "predicted": "yes",
1556
+ "correct": true,
1557
+ "valid": true,
1558
+ "error": null,
1559
+ "probs": {
1560
+ "yes": 0.9158,
1561
+ "no": 0.0842
1562
+ },
1563
+ "latency_s": 0.066
1564
+ },
1565
+ {
1566
+ "id": "original-adequacy-01-1",
1567
+ "tier": "standard",
1568
+ "expected": "yes",
1569
+ "predicted": "yes",
1570
+ "correct": true,
1571
+ "valid": true,
1572
+ "error": null,
1573
+ "probs": {
1574
+ "yes": 0.8119,
1575
+ "no": 0.1881
1576
+ },
1577
+ "latency_s": 0.066
1578
+ },
1579
+ {
1580
+ "id": "original-adequacy-02-0",
1581
+ "tier": "standard",
1582
+ "expected": "no",
1583
+ "predicted": "no",
1584
+ "correct": true,
1585
+ "valid": true,
1586
+ "error": null,
1587
+ "probs": {
1588
+ "yes": 0.0315,
1589
+ "no": 0.9685
1590
+ },
1591
+ "latency_s": 0.067
1592
+ },
1593
+ {
1594
+ "id": "original-adequacy-02-1",
1595
+ "tier": "standard",
1596
+ "expected": "no",
1597
+ "predicted": "no",
1598
+ "correct": true,
1599
+ "valid": true,
1600
+ "error": null,
1601
+ "probs": {
1602
+ "yes": 0.0424,
1603
+ "no": 0.9576
1604
+ },
1605
+ "latency_s": 0.067
1606
+ },
1607
+ {
1608
+ "id": "original-adequacy-03-0",
1609
+ "tier": "standard",
1610
+ "expected": "no",
1611
+ "predicted": "yes",
1612
+ "correct": false,
1613
+ "valid": true,
1614
+ "error": null,
1615
+ "probs": {
1616
+ "yes": 0.8289,
1617
+ "no": 0.1711
1618
+ },
1619
+ "latency_s": 0.067
1620
+ },
1621
+ {
1622
+ "id": "original-adequacy-03-1",
1623
+ "tier": "standard",
1624
+ "expected": "no",
1625
+ "predicted": "yes",
1626
+ "correct": false,
1627
+ "valid": true,
1628
+ "error": null,
1629
+ "probs": {
1630
+ "yes": 0.7936,
1631
+ "no": 0.2064
1632
+ },
1633
+ "latency_s": 0.066
1634
+ },
1635
+ {
1636
+ "id": "original-adequacy-04-0",
1637
+ "tier": "standard",
1638
+ "expected": "yes",
1639
+ "predicted": "yes",
1640
+ "correct": true,
1641
+ "valid": true,
1642
+ "error": null,
1643
+ "probs": {
1644
+ "yes": 0.8768,
1645
+ "no": 0.1232
1646
+ },
1647
+ "latency_s": 0.068
1648
+ },
1649
+ {
1650
+ "id": "original-adequacy-04-1",
1651
+ "tier": "standard",
1652
+ "expected": "yes",
1653
+ "predicted": "yes",
1654
+ "correct": true,
1655
+ "valid": true,
1656
+ "error": null,
1657
+ "probs": {
1658
+ "yes": 0.7312,
1659
+ "no": 0.2688
1660
+ },
1661
+ "latency_s": 0.066
1662
+ },
1663
+ {
1664
+ "id": "original-adequacy-05-0",
1665
+ "tier": "standard",
1666
+ "expected": "no",
1667
+ "predicted": "yes",
1668
+ "correct": false,
1669
+ "valid": true,
1670
+ "error": null,
1671
+ "probs": {
1672
+ "yes": 0.5384,
1673
+ "no": 0.4616
1674
+ },
1675
+ "latency_s": 0.067
1676
+ },
1677
+ {
1678
+ "id": "original-adequacy-05-1",
1679
+ "tier": "standard",
1680
+ "expected": "no",
1681
+ "predicted": "yes",
1682
+ "correct": false,
1683
+ "valid": true,
1684
+ "error": null,
1685
+ "probs": {
1686
+ "yes": 0.6999,
1687
+ "no": 0.3001
1688
+ },
1689
+ "latency_s": 0.068
1690
+ },
1691
+ {
1692
+ "id": "original-adequacy-06-0",
1693
+ "tier": "standard",
1694
+ "expected": "yes",
1695
+ "predicted": "no",
1696
+ "correct": false,
1697
+ "valid": true,
1698
+ "error": null,
1699
+ "probs": {
1700
+ "yes": 0.4236,
1701
+ "no": 0.5764
1702
+ },
1703
+ "latency_s": 0.067
1704
+ },
1705
+ {
1706
+ "id": "original-adequacy-06-1",
1707
+ "tier": "standard",
1708
+ "expected": "yes",
1709
+ "predicted": "yes",
1710
+ "correct": true,
1711
+ "valid": true,
1712
+ "error": null,
1713
+ "probs": {
1714
+ "yes": 0.595,
1715
+ "no": 0.405
1716
+ },
1717
+ "latency_s": 0.067
1718
+ },
1719
+ {
1720
+ "id": "original-routing-01-0",
1721
+ "tier": "standard",
1722
+ "expected": "math",
1723
+ "predicted": "math",
1724
+ "correct": true,
1725
+ "valid": true,
1726
+ "error": null,
1727
+ "probs": {
1728
+ "coding": 0.0024,
1729
+ "coding_agent": 0.0004,
1730
+ "document": 0.0011,
1731
+ "general": 0.0141,
1732
+ "math": 0.9805,
1733
+ "tools": 0.0016
1734
+ },
1735
+ "latency_s": 0.067
1736
+ },
1737
+ {
1738
+ "id": "original-routing-01-1",
1739
+ "tier": "standard",
1740
+ "expected": "math",
1741
+ "predicted": "math",
1742
+ "correct": true,
1743
+ "valid": true,
1744
+ "error": null,
1745
+ "probs": {
1746
+ "coding": 0.0019,
1747
+ "coding_agent": 0.0003,
1748
+ "document": 0.0015,
1749
+ "general": 0.0157,
1750
+ "math": 0.9794,
1751
+ "tools": 0.0012
1752
+ },
1753
+ "latency_s": 0.068
1754
+ },
1755
+ {
1756
+ "id": "original-routing-02-0",
1757
+ "tier": "standard",
1758
+ "expected": "coding",
1759
+ "predicted": "coding",
1760
+ "correct": true,
1761
+ "valid": true,
1762
+ "error": null,
1763
+ "probs": {
1764
+ "coding": 0.9695,
1765
+ "coding_agent": 0.0192,
1766
+ "document": 0.001,
1767
+ "general": 0.0091,
1768
+ "math": 0.0005,
1769
+ "tools": 0.0007
1770
+ },
1771
+ "latency_s": 0.071
1772
+ },
1773
+ {
1774
+ "id": "original-routing-02-1",
1775
+ "tier": "standard",
1776
+ "expected": "coding",
1777
+ "predicted": "coding",
1778
+ "correct": true,
1779
+ "valid": true,
1780
+ "error": null,
1781
+ "probs": {
1782
+ "coding": 0.9822,
1783
+ "coding_agent": 0.0078,
1784
+ "document": 0.001,
1785
+ "general": 0.0078,
1786
+ "math": 0.0005,
1787
+ "tools": 0.0006
1788
+ },
1789
+ "latency_s": 0.071
1790
+ },
1791
+ {
1792
+ "id": "original-routing-03-0",
1793
+ "tier": "standard",
1794
+ "expected": "coding_agent",
1795
+ "predicted": "coding_agent",
1796
+ "correct": true,
1797
+ "valid": true,
1798
+ "error": null,
1799
+ "probs": {
1800
+ "coding": 0.0014,
1801
+ "coding_agent": 0.9964,
1802
+ "document": 0.0004,
1803
+ "general": 0.001,
1804
+ "math": 0.0002,
1805
+ "tools": 0.0006
1806
+ },
1807
+ "latency_s": 0.077
1808
+ },
1809
+ {
1810
+ "id": "original-routing-03-1",
1811
+ "tier": "standard",
1812
+ "expected": "coding_agent",
1813
+ "predicted": "coding_agent",
1814
+ "correct": true,
1815
+ "valid": true,
1816
+ "error": null,
1817
+ "probs": {
1818
+ "coding": 0.001,
1819
+ "coding_agent": 0.9959,
1820
+ "document": 0.0004,
1821
+ "general": 0.0018,
1822
+ "math": 0.0002,
1823
+ "tools": 0.0006
1824
+ },
1825
+ "latency_s": 0.07
1826
+ },
1827
+ {
1828
+ "id": "original-routing-04-0",
1829
+ "tier": "standard",
1830
+ "expected": "document",
1831
+ "predicted": "document",
1832
+ "correct": true,
1833
+ "valid": true,
1834
+ "error": null,
1835
+ "probs": {
1836
+ "coding": 0.0023,
1837
+ "coding_agent": 0.0029,
1838
+ "document": 0.8015,
1839
+ "general": 0.1881,
1840
+ "math": 0.0009,
1841
+ "tools": 0.0044
1842
+ },
1843
+ "latency_s": 0.068
1844
+ },
1845
+ {
1846
+ "id": "original-routing-04-1",
1847
+ "tier": "standard",
1848
+ "expected": "document",
1849
+ "predicted": "document",
1850
+ "correct": true,
1851
+ "valid": true,
1852
+ "error": null,
1853
+ "probs": {
1854
+ "coding": 0.0011,
1855
+ "coding_agent": 0.0013,
1856
+ "document": 0.9304,
1857
+ "general": 0.0635,
1858
+ "math": 0.0004,
1859
+ "tools": 0.0033
1860
+ },
1861
+ "latency_s": 0.068
1862
+ },
1863
+ {
1864
+ "id": "original-routing-05-0",
1865
+ "tier": "standard",
1866
+ "expected": "tools",
1867
+ "predicted": "tools",
1868
+ "correct": true,
1869
+ "valid": true,
1870
+ "error": null,
1871
+ "probs": {
1872
+ "coding": 0.0024,
1873
+ "coding_agent": 0.0011,
1874
+ "document": 0.0026,
1875
+ "general": 0.2529,
1876
+ "math": 0.0009,
1877
+ "tools": 0.7401
1878
+ },
1879
+ "latency_s": 0.067
1880
+ },
1881
+ {
1882
+ "id": "original-routing-05-1",
1883
+ "tier": "standard",
1884
+ "expected": "tools",
1885
+ "predicted": "general",
1886
+ "correct": false,
1887
+ "valid": true,
1888
+ "error": null,
1889
+ "probs": {
1890
+ "coding": 0.0014,
1891
+ "coding_agent": 0.0009,
1892
+ "document": 0.0017,
1893
+ "general": 0.9215,
1894
+ "math": 0.0006,
1895
+ "tools": 0.0739
1896
+ },
1897
+ "latency_s": 0.068
1898
+ },
1899
+ {
1900
+ "id": "original-routing-06-0",
1901
+ "tier": "standard",
1902
+ "expected": "general",
1903
+ "predicted": "general",
1904
+ "correct": true,
1905
+ "valid": true,
1906
+ "error": null,
1907
+ "probs": {
1908
+ "coding": 0.0009,
1909
+ "coding_agent": 0.0006,
1910
+ "document": 0.003,
1911
+ "general": 0.994,
1912
+ "math": 0.0005,
1913
+ "tools": 0.0009
1914
+ },
1915
+ "latency_s": 0.067
1916
+ },
1917
+ {
1918
+ "id": "original-routing-06-1",
1919
+ "tier": "standard",
1920
+ "expected": "general",
1921
+ "predicted": "general",
1922
+ "correct": true,
1923
+ "valid": true,
1924
+ "error": null,
1925
+ "probs": {
1926
+ "coding": 0.0012,
1927
+ "coding_agent": 0.0007,
1928
+ "document": 0.003,
1929
+ "general": 0.9934,
1930
+ "math": 0.0005,
1931
+ "tools": 0.0011
1932
+ },
1933
+ "latency_s": 0.067
1934
+ },
1935
+ {
1936
+ "id": "hard-opus-a-long_policy-01",
1937
+ "tier": "hard",
1938
+ "expected": "pay_subject_to_15000_sublimit",
1939
+ "predicted": "pay_subject_to_15000_sublimit",
1940
+ "correct": true,
1941
+ "valid": true,
1942
+ "error": null,
1943
+ "probs": {
1944
+ "deny_repeated_seepage": 0.0006,
1945
+ "deny_vacancy_exclusion": 0.0382,
1946
+ "pay_full_estimate_less_deductible": 0.0006,
1947
+ "pay_subject_to_10000_sublimit": 0.0022,
1948
+ "pay_subject_to_15000_sublimit": 0.9584
1949
+ },
1950
+ "latency_s": 0.156
1951
+ },
1952
+ {
1953
+ "id": "hard-opus-a-long_policy-04",
1954
+ "tier": "hard",
1955
+ "expected": "cfo",
1956
+ "predicted": "vp_and_finance_director",
1957
+ "correct": false,
1958
+ "valid": true,
1959
+ "error": null,
1960
+ "probs": {
1961
+ "budget_holder": 0.0023,
1962
+ "cfo": 0.0023,
1963
+ "department_head": 0.1888,
1964
+ "executive_committee": 0.0018,
1965
+ "vp_and_finance_director": 0.8048
1966
+ },
1967
+ "latency_s": 0.089
1968
+ },
1969
+ {
1970
+ "id": "hard-opus-a-long_policy-08",
1971
+ "tier": "hard",
1972
+ "expected": "parts_covered_labour_charged",
1973
+ "predicted": "parts_covered_labour_charged",
1974
+ "correct": true,
1975
+ "valid": true,
1976
+ "error": null,
1977
+ "probs": {
1978
+ "full_repair_no_charge": 0.0342,
1979
+ "not_covered": 0.008,
1980
+ "parts_covered_labour_charged": 0.9549,
1981
+ "pro_rata_replacement_credit": 0.0029
1982
+ },
1983
+ "latency_s": 0.083
1984
+ },
1985
+ {
1986
+ "id": "hard-opus-a-long_policy-09",
1987
+ "tier": "hard",
1988
+ "expected": "license_required",
1989
+ "predicted": "license_exception_lvs",
1990
+ "correct": false,
1991
+ "valid": true,
1992
+ "error": null,
1993
+ "probs": {
1994
+ "license_exception_lvs": 0.984,
1995
+ "license_required": 0.0074,
1996
+ "no_license_required": 0.0063,
1997
+ "shipment_prohibited": 0.0023
1998
+ },
1999
+ "latency_s": 0.085
2000
+ },
2001
+ {
2002
+ "id": "hard-opus-a-long_policy-11",
2003
+ "tier": "hard",
2004
+ "expected": "yes",
2005
+ "predicted": "no",
2006
+ "correct": false,
2007
+ "valid": true,
2008
+ "error": null,
2009
+ "probs": {
2010
+ "yes": 0.0053,
2011
+ "no": 0.9947
2012
+ },
2013
+ "latency_s": 0.081
2014
+ },
2015
+ {
2016
+ "id": "hard-opus-a-long_policy-13",
2017
+ "tier": "hard",
2018
+ "expected": "yes",
2019
+ "predicted": "no",
2020
+ "correct": false,
2021
+ "valid": true,
2022
+ "error": null,
2023
+ "probs": {
2024
+ "yes": 0.1191,
2025
+ "no": 0.8809
2026
+ },
2027
+ "latency_s": 0.083
2028
+ },
2029
+ {
2030
+ "id": "hard-opus-a-long_policy-17",
2031
+ "tier": "hard",
2032
+ "expected": "database_oncall",
2033
+ "predicted": "database_oncall",
2034
+ "correct": true,
2035
+ "valid": true,
2036
+ "error": null,
2037
+ "probs": {
2038
+ "database_oncall": 0.7764,
2039
+ "no_page_ticket_only": 0.1726,
2040
+ "platform_oncall": 0.0069,
2041
+ "security_incident_response": 0.0014,
2042
+ "vendor_support": 0.0427
2043
+ },
2044
+ "latency_s": 0.081
2045
+ },
2046
+ {
2047
+ "id": "hard-opus-a-long_policy-19",
2048
+ "tier": "hard",
2049
+ "expected": "no",
2050
+ "predicted": "no",
2051
+ "correct": true,
2052
+ "valid": true,
2053
+ "error": null,
2054
+ "probs": {
2055
+ "yes": 0.0261,
2056
+ "no": 0.9739
2057
+ },
2058
+ "latency_s": 0.081
2059
+ },
2060
+ {
2061
+ "id": "hard-opus-a-probability-03",
2062
+ "tier": "hard",
2063
+ "expected": "yes",
2064
+ "predicted": "yes",
2065
+ "correct": true,
2066
+ "valid": true,
2067
+ "error": null,
2068
+ "probs": {
2069
+ "yes": 0.7936,
2070
+ "no": 0.2064
2071
+ },
2072
+ "latency_s": 0.07
2073
+ },
2074
+ {
2075
+ "id": "hard-opus-a-probability-04",
2076
+ "tier": "hard",
2077
+ "expected": "on_time",
2078
+ "predicted": "on_time",
2079
+ "correct": true,
2080
+ "valid": true,
2081
+ "error": null,
2082
+ "probs": {
2083
+ "early": 0.0134,
2084
+ "late": 0.1031,
2085
+ "on_time": 0.8835
2086
+ },
2087
+ "latency_s": 0.072
2088
+ },
2089
+ {
2090
+ "id": "hard-opus-a-probability-07",
2091
+ "tier": "hard",
2092
+ "expected": "escalated",
2093
+ "predicted": "escalated",
2094
+ "correct": true,
2095
+ "valid": true,
2096
+ "error": null,
2097
+ "probs": {
2098
+ "escalated": 0.9841,
2099
+ "reopened": 0.0067,
2100
+ "resolved_first_contact": 0.0092
2101
+ },
2102
+ "latency_s": 0.07
2103
+ },
2104
+ {
2105
+ "id": "hard-opus-a-probability-08",
2106
+ "tier": "hard",
2107
+ "expected": "no",
2108
+ "predicted": "yes",
2109
+ "correct": false,
2110
+ "valid": true,
2111
+ "error": null,
2112
+ "probs": {
2113
+ "yes": 0.7312,
2114
+ "no": 0.2688
2115
+ },
2116
+ "latency_s": 0.068
2117
+ },
2118
+ {
2119
+ "id": "hard-opus-a-temporal_numeric-03",
2120
+ "tier": "hard",
2121
+ "expected": "yes",
2122
+ "predicted": "no",
2123
+ "correct": false,
2124
+ "valid": true,
2125
+ "error": null,
2126
+ "probs": {
2127
+ "yes": 0.3083,
2128
+ "no": 0.6917
2129
+ },
2130
+ "latency_s": 0.069
2131
+ },
2132
+ {
2133
+ "id": "hard-opus-a-temporal_numeric-06",
2134
+ "tier": "hard",
2135
+ "expected": "no",
2136
+ "predicted": "no",
2137
+ "correct": true,
2138
+ "valid": true,
2139
+ "error": null,
2140
+ "probs": {
2141
+ "yes": 0.0785,
2142
+ "no": 0.9215
2143
+ },
2144
+ "latency_s": 0.068
2145
+ },
2146
+ {
2147
+ "id": "hard-opus-a-temporal_numeric-07",
2148
+ "tier": "hard",
2149
+ "expected": "sep_26",
2150
+ "predicted": "sep_24",
2151
+ "correct": false,
2152
+ "valid": true,
2153
+ "error": null,
2154
+ "probs": {
2155
+ "sep_24": 0.5907,
2156
+ "sep_25": 0.0111,
2157
+ "sep_26": 0.0065,
2158
+ "sep_27": 0.0072,
2159
+ "sep_28": 0.3844
2160
+ },
2161
+ "latency_s": 0.07
2162
+ },
2163
+ {
2164
+ "id": "hard-opus-a-temporal_numeric-09",
2165
+ "tier": "hard",
2166
+ "expected": "no",
2167
+ "predicted": "no",
2168
+ "correct": true,
2169
+ "valid": true,
2170
+ "error": null,
2171
+ "probs": {
2172
+ "yes": 0.1191,
2173
+ "no": 0.8809
2174
+ },
2175
+ "latency_s": 0.07
2176
+ },
2177
+ {
2178
+ "id": "hard-opus-a-temporal_numeric-12",
2179
+ "tier": "hard",
2180
+ "expected": "1",
2181
+ "predicted": "2",
2182
+ "correct": false,
2183
+ "valid": true,
2184
+ "error": null,
2185
+ "probs": {
2186
+ "0": 0.2126,
2187
+ "1": 0.3652,
2188
+ "2": 0.418,
2189
+ "3": 0.0042
2190
+ },
2191
+ "latency_s": 0.073
2192
+ },
2193
+ {
2194
+ "id": "hard-opus-b-ambiguous-02",
2195
+ "tier": "hard",
2196
+ "expected": "real_incident",
2197
+ "predicted": "real_incident",
2198
+ "correct": true,
2199
+ "valid": true,
2200
+ "error": null,
2201
+ "probs": {
2202
+ "cannot_determine": 0.006,
2203
+ "false_positive": 0.1973,
2204
+ "real_incident": 0.7967
2205
+ },
2206
+ "latency_s": 0.069
2207
+ },
2208
+ {
2209
+ "id": "hard-opus-b-ambiguous-03",
2210
+ "tier": "hard",
2211
+ "expected": "time_barred",
2212
+ "predicted": "time_barred",
2213
+ "correct": true,
2214
+ "valid": true,
2215
+ "error": null,
2216
+ "probs": {
2217
+ "insufficient_information": 0.0049,
2218
+ "time_barred": 0.9377,
2219
+ "within_limitation": 0.0575
2220
+ },
2221
+ "latency_s": 0.069
2222
+ },
2223
+ {
2224
+ "id": "hard-opus-b-ambiguous-07",
2225
+ "tier": "hard",
2226
+ "expected": "within_policy",
2227
+ "predicted": "insufficient_information",
2228
+ "correct": false,
2229
+ "valid": true,
2230
+ "error": null,
2231
+ "probs": {
2232
+ "insufficient_information": 0.5982,
2233
+ "over_limit": 0.3893,
2234
+ "within_policy": 0.0125
2235
+ },
2236
+ "latency_s": 0.078
2237
+ },
2238
+ {
2239
+ "id": "hard-opus-b-ambiguous-09",
2240
+ "tier": "hard",
2241
+ "expected": "legitimate",
2242
+ "predicted": "legitimate",
2243
+ "correct": true,
2244
+ "valid": true,
2245
+ "error": null,
2246
+ "probs": {
2247
+ "compromised": 0.0903,
2248
+ "inconclusive": 0.0014,
2249
+ "legitimate": 0.9083
2250
+ },
2251
+ "latency_s": 0.07
2252
+ },
2253
+ {
2254
+ "id": "hard-opus-b-ambiguous-10",
2255
+ "tier": "hard",
2256
+ "expected": "ask_user_which_alex",
2257
+ "predicted": "send_to_alex_moreno",
2258
+ "correct": false,
2259
+ "valid": true,
2260
+ "error": null,
2261
+ "probs": {
2262
+ "ask_user_which_alex": 0.0295,
2263
+ "send_to_alex_moreno": 0.9677,
2264
+ "send_to_alexandra_lind": 0.0028
2265
+ },
2266
+ "latency_s": 0.069
2267
+ },
2268
+ {
2269
+ "id": "hard-opus-b-ambiguous-11",
2270
+ "tier": "hard",
2271
+ "expected": "cannot_determine",
2272
+ "predicted": "cannot_determine",
2273
+ "correct": true,
2274
+ "valid": true,
2275
+ "error": null,
2276
+ "probs": {
2277
+ "cannot_determine": 0.9992,
2278
+ "grade_correct": 0.0006,
2279
+ "grade_should_be_higher": 0.0002
2280
+ },
2281
+ "latency_s": 0.069
2282
+ },
2283
+ {
2284
+ "id": "hard-opus-b-ambiguous-13",
2285
+ "tier": "hard",
2286
+ "expected": "cannot_determine",
2287
+ "predicted": "cannot_determine",
2288
+ "correct": true,
2289
+ "valid": true,
2290
+ "error": null,
2291
+ "probs": {
2292
+ "cannot_determine": 0.9988,
2293
+ "charge_data_team": 0.0004,
2294
+ "charge_platform_team": 0.0005,
2295
+ "split_50_50": 0.0003
2296
+ },
2297
+ "latency_s": 0.071
2298
+ },
2299
+ {
2300
+ "id": "hard-opus-b-multi_hop-03",
2301
+ "tier": "hard",
2302
+ "expected": "tier3_cfo",
2303
+ "predicted": "tier3_cfo",
2304
+ "correct": true,
2305
+ "valid": true,
2306
+ "error": null,
2307
+ "probs": {
2308
+ "tier1_team_lead": 0.1636,
2309
+ "tier2_department_head": 0.0364,
2310
+ "tier3_cfo": 0.7763,
2311
+ "tier4_board": 0.0237
2312
+ },
2313
+ "latency_s": 0.09
2314
+ },
2315
+ {
2316
+ "id": "hard-opus-b-multi_hop-04",
2317
+ "tier": "hard",
2318
+ "expected": "bjorn",
2319
+ "predicted": "no_page_ticket_only",
2320
+ "correct": false,
2321
+ "valid": true,
2322
+ "error": null,
2323
+ "probs": {
2324
+ "ana": 0.0148,
2325
+ "bjorn": 0.0253,
2326
+ "chen": 0.0298,
2327
+ "dmitri": 0.0019,
2328
+ "esra": 0.0028,
2329
+ "no_page_ticket_only": 0.9253
2330
+ },
2331
+ "latency_s": 0.081
2332
+ },
2333
+ {
2334
+ "id": "hard-opus-b-multi_hop-05",
2335
+ "tier": "hard",
2336
+ "expected": "2",
2337
+ "predicted": "2",
2338
+ "correct": true,
2339
+ "valid": true,
2340
+ "error": null,
2341
+ "probs": {
2342
+ "0": 0.0036,
2343
+ "1": 0.0491,
2344
+ "2": 0.9365,
2345
+ "3": 0.005,
2346
+ "4": 0.0058
2347
+ },
2348
+ "latency_s": 0.369
2349
+ },
2350
+ {
2351
+ "id": "hard-opus-b-multi_hop-07",
2352
+ "tier": "hard",
2353
+ "expected": "credit_20_percent",
2354
+ "predicted": "credit_50_percent",
2355
+ "correct": false,
2356
+ "valid": true,
2357
+ "error": null,
2358
+ "probs": {
2359
+ "credit_10_percent": 0.0127,
2360
+ "credit_20_percent": 0.0074,
2361
+ "credit_25_percent": 0.0876,
2362
+ "credit_50_percent": 0.7506,
2363
+ "credit_full_month": 0.1276,
2364
+ "no_credit": 0.0141
2365
+ },
2366
+ "latency_s": 0.088
2367
+ },
2368
+ {
2369
+ "id": "hard-opus-b-multi_hop-08",
2370
+ "tier": "hard",
2371
+ "expected": "no",
2372
+ "predicted": "no",
2373
+ "correct": true,
2374
+ "valid": true,
2375
+ "error": null,
2376
+ "probs": {
2377
+ "yes": 0.1605,
2378
+ "no": 0.8395
2379
+ },
2380
+ "latency_s": 0.076
2381
+ },
2382
+ {
2383
+ "id": "hard-opus-b-probability-01",
2384
+ "tier": "hard",
2385
+ "expected": "merchant_wins",
2386
+ "predicted": "merchant_wins",
2387
+ "correct": true,
2388
+ "valid": true,
2389
+ "error": null,
2390
+ "probs": {
2391
+ "customer_wins": 0.0007,
2392
+ "customer_withdraws": 0.0012,
2393
+ "merchant_wins": 0.9981
2394
+ },
2395
+ "latency_s": 0.073
2396
+ },
2397
+ {
2398
+ "id": "hard-opus-b-probability-02",
2399
+ "tier": "hard",
2400
+ "expected": "upstream_provider",
2401
+ "predicted": "upstream_provider",
2402
+ "correct": true,
2403
+ "valid": true,
2404
+ "error": null,
2405
+ "probs": {
2406
+ "bad_push": 0.0266,
2407
+ "database_hardware": 0.0029,
2408
+ "upstream_provider": 0.9705
2409
+ },
2410
+ "latency_s": 0.069
2411
+ },
2412
+ {
2413
+ "id": "hard-opus-b-probability-03",
2414
+ "tier": "hard",
2415
+ "expected": "yes",
2416
+ "predicted": "yes",
2417
+ "correct": true,
2418
+ "valid": true,
2419
+ "error": null,
2420
+ "probs": {
2421
+ "yes": 0.8726,
2422
+ "no": 0.1274
2423
+ },
2424
+ "latency_s": 0.071
2425
+ },
2426
+ {
2427
+ "id": "hard-opus-b-probability-04",
2428
+ "tier": "hard",
2429
+ "expected": "attended",
2430
+ "predicted": "attended",
2431
+ "correct": true,
2432
+ "valid": true,
2433
+ "error": null,
2434
+ "probs": {
2435
+ "attended": 0.9841,
2436
+ "late_cancellation": 0.0103,
2437
+ "no_show": 0.0057
2438
+ },
2439
+ "latency_s": 0.071
2440
+ },
2441
+ {
2442
+ "id": "hard-opus-b-probability-06",
2443
+ "tier": "hard",
2444
+ "expected": "upheld",
2445
+ "predicted": "upheld",
2446
+ "correct": true,
2447
+ "valid": true,
2448
+ "error": null,
2449
+ "probs": {
2450
+ "modified": 0.0013,
2451
+ "overturned": 0.0103,
2452
+ "upheld": 0.9884
2453
+ },
2454
+ "latency_s": 0.071
2455
+ },
2456
+ {
2457
+ "id": "hard-opus-b-tradeoff-01",
2458
+ "tier": "hard",
2459
+ "expected": "escalate_to_engineering",
2460
+ "predicted": "escalate_to_security",
2461
+ "correct": false,
2462
+ "valid": true,
2463
+ "error": null,
2464
+ "probs": {
2465
+ "escalate_to_engineering": 0.0034,
2466
+ "escalate_to_security": 0.9962,
2467
+ "page_oncall": 0.0002,
2468
+ "process_refund": 0.0001,
2469
+ "send_kb_article": 0.0001
2470
+ },
2471
+ "latency_s": 0.071
2472
+ },
2473
+ {
2474
+ "id": "hard-opus-b-tradeoff-03",
2475
+ "tier": "hard",
2476
+ "expected": "order_b",
2477
+ "predicted": "order_b",
2478
+ "correct": true,
2479
+ "valid": true,
2480
+ "error": null,
2481
+ "probs": {
2482
+ "order_a": 0.0248,
2483
+ "order_b": 0.9566,
2484
+ "order_c": 0.0105,
2485
+ "order_d": 0.008
2486
+ },
2487
+ "latency_s": 0.07
2488
+ },
2489
+ {
2490
+ "id": "hard-opus-b-tradeoff-06",
2491
+ "tier": "hard",
2492
+ "expected": "yes",
2493
+ "predicted": "yes",
2494
+ "correct": true,
2495
+ "valid": true,
2496
+ "error": null,
2497
+ "probs": {
2498
+ "yes": 0.8888,
2499
+ "no": 0.1112
2500
+ },
2501
+ "latency_s": 0.07
2502
+ },
2503
+ {
2504
+ "id": "hard-opus-b-tradeoff-07",
2505
+ "tier": "hard",
2506
+ "expected": "p0_fix_24h",
2507
+ "predicted": "p0_fix_24h",
2508
+ "correct": true,
2509
+ "valid": true,
2510
+ "error": null,
2511
+ "probs": {
2512
+ "out_of_scope": 0.0005,
2513
+ "p0_fix_24h": 0.9513,
2514
+ "p1_fix_7d": 0.001,
2515
+ "p2_fix_30d": 0.047,
2516
+ "p3_backlog": 0.0002
2517
+ },
2518
+ "latency_s": 0.07
2519
+ },
2520
+ {
2521
+ "id": "hard-opus-b-tradeoff-08",
2522
+ "tier": "hard",
2523
+ "expected": "yes",
2524
+ "predicted": "no",
2525
+ "correct": false,
2526
+ "valid": true,
2527
+ "error": null,
2528
+ "probs": {
2529
+ "yes": 0.452,
2530
+ "no": 0.548
2531
+ },
2532
+ "latency_s": 0.07
2533
+ },
2534
+ {
2535
+ "id": "hard-opus-b-tradeoff-12",
2536
+ "tier": "hard",
2537
+ "expected": "refund_now",
2538
+ "predicted": "refund_now",
2539
+ "correct": true,
2540
+ "valid": true,
2541
+ "error": null,
2542
+ "probs": {
2543
+ "deny_refund": 0.0002,
2544
+ "escalate_to_fraud_team": 0.0001,
2545
+ "offer_store_credit": 0.0001,
2546
+ "refund_now": 0.9996,
2547
+ "wait_for_inspection": 0.0001
2548
+ },
2549
+ "latency_s": 0.074
2550
+ },
2551
+ {
2552
+ "id": "hard-opus-c-long_policy-02",
2553
+ "tier": "hard",
2554
+ "expected": "erase_but_retain_transaction_records",
2555
+ "predicted": "restrict_all_pending_legal_hold",
2556
+ "correct": false,
2557
+ "valid": true,
2558
+ "error": null,
2559
+ "probs": {
2560
+ "erase_all": 0.001,
2561
+ "erase_but_retain_transaction_records": 0.047,
2562
+ "erase_marketing_only": 0.0007,
2563
+ "reject_unverified_request": 0.0004,
2564
+ "restrict_all_pending_legal_hold": 0.9508
2565
+ },
2566
+ "latency_s": 0.118
2567
+ },
2568
+ {
2569
+ "id": "hard-opus-c-long_policy-03",
2570
+ "tier": "hard",
2571
+ "expected": "eur_400",
2572
+ "predicted": "eur_0",
2573
+ "correct": false,
2574
+ "valid": true,
2575
+ "error": null,
2576
+ "probs": {
2577
+ "eur_0": 0.915,
2578
+ "eur_150": 0.0118,
2579
+ "eur_200": 0.0095,
2580
+ "eur_300": 0.0182,
2581
+ "eur_400": 0.009,
2582
+ "eur_600": 0.0365
2583
+ },
2584
+ "latency_s": 0.117
2585
+ },
2586
+ {
2587
+ "id": "hard-opus-c-long_policy-04",
2588
+ "tier": "hard",
2589
+ "expected": "yes",
2590
+ "predicted": "yes",
2591
+ "correct": true,
2592
+ "valid": true,
2593
+ "error": null,
2594
+ "probs": {
2595
+ "yes": 0.8177,
2596
+ "no": 0.1823
2597
+ },
2598
+ "latency_s": 0.105
2599
+ },
2600
+ {
2601
+ "id": "hard-opus-c-long_policy-05",
2602
+ "tier": "hard",
2603
+ "expected": "1",
2604
+ "predicted": "2",
2605
+ "correct": false,
2606
+ "valid": true,
2607
+ "error": null,
2608
+ "probs": {
2609
+ "0": 0.0172,
2610
+ "1": 0.3344,
2611
+ "2": 0.5975,
2612
+ "3": 0.0508
2613
+ },
2614
+ "latency_s": 0.372
2615
+ },
2616
+ {
2617
+ "id": "hard-opus-c-long_policy-08",
2618
+ "tier": "hard",
2619
+ "expected": "1",
2620
+ "predicted": "1",
2621
+ "correct": true,
2622
+ "valid": true,
2623
+ "error": null,
2624
+ "probs": {
2625
+ "0": 0.0041,
2626
+ "1": 0.9895,
2627
+ "2": 0.0039,
2628
+ "3": 0.0026
2629
+ },
2630
+ "latency_s": 0.378
2631
+ },
2632
+ {
2633
+ "id": "hard-opus-c-long_policy-10",
2634
+ "tier": "hard",
2635
+ "expected": "credit_minus_restocking_fee",
2636
+ "predicted": "credit_minus_restocking_fee",
2637
+ "correct": true,
2638
+ "valid": true,
2639
+ "error": null,
2640
+ "probs": {
2641
+ "credit_minus_restocking_fee": 0.9098,
2642
+ "full_credit": 0.077,
2643
+ "reject_return": 0.0095,
2644
+ "repair_or_replace_only": 0.0038
2645
+ },
2646
+ "latency_s": 0.121
2647
+ },
2648
+ {
2649
+ "id": "hard-opus-c-long_policy-11",
2650
+ "tier": "hard",
2651
+ "expected": "no",
2652
+ "predicted": "no",
2653
+ "correct": true,
2654
+ "valid": true,
2655
+ "error": null,
2656
+ "probs": {
2657
+ "yes": 0.0785,
2658
+ "no": 0.9215
2659
+ },
2660
+ "latency_s": 0.105
2661
+ },
2662
+ {
2663
+ "id": "hard-opus-c-probability-03",
2664
+ "tier": "hard",
2665
+ "expected": "kuznets_technik",
2666
+ "predicted": "halvorsen_polymer",
2667
+ "correct": false,
2668
+ "valid": true,
2669
+ "error": null,
2670
+ "probs": {
2671
+ "halvorsen_polymer": 0.985,
2672
+ "kuznets_technik": 0.0083,
2673
+ "ortega_seal_works": 0.0067
2674
+ },
2675
+ "latency_s": 0.071
2676
+ },
2677
+ {
2678
+ "id": "hard-opus-c-temporal_numeric-02",
2679
+ "tier": "hard",
2680
+ "expected": "yes",
2681
+ "predicted": "no",
2682
+ "correct": false,
2683
+ "valid": true,
2684
+ "error": null,
2685
+ "probs": {
2686
+ "yes": 0.0271,
2687
+ "no": 0.9729
2688
+ },
2689
+ "latency_s": 0.071
2690
+ },
2691
+ {
2692
+ "id": "hard-opus-c-temporal_numeric-03",
2693
+ "tier": "hard",
2694
+ "expected": "eur_692_00",
2695
+ "predicted": "eur_693_90",
2696
+ "correct": false,
2697
+ "valid": true,
2698
+ "error": null,
2699
+ "probs": {
2700
+ "eur_617_00": 0.0525,
2701
+ "eur_688_00": 0.0401,
2702
+ "eur_692_00": 0.2491,
2703
+ "eur_693_90": 0.6547,
2704
+ "eur_812_00": 0.0036
2705
+ },
2706
+ "latency_s": 0.073
2707
+ },
2708
+ {
2709
+ "id": "hard-opus-c-temporal_numeric-04",
2710
+ "tier": "hard",
2711
+ "expected": "usd_2335_00",
2712
+ "predicted": "usd_2303_01",
2713
+ "correct": false,
2714
+ "valid": true,
2715
+ "error": null,
2716
+ "probs": {
2717
+ "usd_2222_50": 0.0175,
2718
+ "usd_2303_01": 0.5751,
2719
+ "usd_2303_33": 0.3547,
2720
+ "usd_2335_00": 0.0392,
2721
+ "usd_2340_00": 0.0134
2722
+ },
2723
+ "latency_s": 0.07
2724
+ },
2725
+ {
2726
+ "id": "hard-opus-c-temporal_numeric-06",
2727
+ "tier": "hard",
2728
+ "expected": "sep_18_book_set",
2729
+ "predicted": "none_declined",
2730
+ "correct": false,
2731
+ "valid": true,
2732
+ "error": null,
2733
+ "probs": {
2734
+ "none_declined": 0.9613,
2735
+ "sep_15_office_chair": 0.0138,
2736
+ "sep_16_taxi": 0.0059,
2737
+ "sep_17_parking": 0.0118,
2738
+ "sep_18_book_set": 0.004,
2739
+ "sep_21_client_dinner": 0.0032
2740
+ },
2741
+ "latency_s": 0.07
2742
+ },
2743
+ {
2744
+ "id": "hard-opus-c-temporal_numeric-08",
2745
+ "tier": "hard",
2746
+ "expected": "eur_498_81",
2747
+ "predicted": "eur_513_69",
2748
+ "correct": false,
2749
+ "valid": true,
2750
+ "error": null,
2751
+ "probs": {
2752
+ "eur_497_40": 0.1137,
2753
+ "eur_498_81": 0.1569,
2754
+ "eur_506_06": 0.2052,
2755
+ "eur_513_69": 0.3908,
2756
+ "eur_532_01": 0.1335
2757
+ },
2758
+ "latency_s": 0.071
2759
+ },
2760
+ {
2761
+ "id": "hard-opus-c-temporal_numeric-12",
2762
+ "tier": "hard",
2763
+ "expected": "expired_30_month_cap",
2764
+ "predicted": "expired_30_month_cap",
2765
+ "correct": true,
2766
+ "valid": true,
2767
+ "error": null,
2768
+ "probs": {
2769
+ "covered": 0.0022,
2770
+ "expired_24_month_term": 0.0098,
2771
+ "expired_30_month_cap": 0.9881
2772
+ },
2773
+ "latency_s": 0.074
2774
+ },
2775
+ {
2776
+ "id": "hard-sol-a-adversarial-01",
2777
+ "tier": "hard",
2778
+ "expected": "deny_missing_approval",
2779
+ "predicted": "deny_missing_approval",
2780
+ "correct": true,
2781
+ "valid": true,
2782
+ "error": null,
2783
+ "probs": {
2784
+ "deny_inactive_user": 0.0001,
2785
+ "deny_missing_approval": 0.9991,
2786
+ "grant_access": 0.0003,
2787
+ "grant_owner_exception": 0.0005
2788
+ },
2789
+ "latency_s": 0.069
2790
+ },
2791
+ {
2792
+ "id": "hard-sol-a-adversarial-06",
2793
+ "tier": "hard",
2794
+ "expected": "no",
2795
+ "predicted": "no",
2796
+ "correct": true,
2797
+ "valid": true,
2798
+ "error": null,
2799
+ "probs": {
2800
+ "yes": 0.0027,
2801
+ "no": 0.9973
2802
+ },
2803
+ "latency_s": 0.068
2804
+ },
2805
+ {
2806
+ "id": "hard-sol-a-adversarial-07",
2807
+ "tier": "hard",
2808
+ "expected": "retain_until_2031",
2809
+ "predicted": "retain_until_2031",
2810
+ "correct": true,
2811
+ "valid": true,
2812
+ "error": null,
2813
+ "probs": {
2814
+ "delete_now": 0.0032,
2815
+ "needs_signature_review": 0.0001,
2816
+ "retain_indefinitely": 0.0001,
2817
+ "retain_until_2031": 0.9966
2818
+ },
2819
+ "latency_s": 0.068
2820
+ },
2821
+ {
2822
+ "id": "hard-sol-a-adversarial-08",
2823
+ "tier": "hard",
2824
+ "expected": "yes",
2825
+ "predicted": "yes",
2826
+ "correct": true,
2827
+ "valid": true,
2828
+ "error": null,
2829
+ "probs": {
2830
+ "yes": 0.9951,
2831
+ "no": 0.0049
2832
+ },
2833
+ "latency_s": 0.067
2834
+ },
2835
+ {
2836
+ "id": "hard-sol-a-adversarial-09",
2837
+ "tier": "hard",
2838
+ "expected": "2",
2839
+ "predicted": "2",
2840
+ "correct": true,
2841
+ "valid": true,
2842
+ "error": null,
2843
+ "probs": {
2844
+ "0": 0.0028,
2845
+ "1": 0.0084,
2846
+ "2": 0.9843,
2847
+ "3": 0.0045
2848
+ },
2849
+ "latency_s": 0.07
2850
+ },
2851
+ {
2852
+ "id": "hard-sol-a-adversarial-11",
2853
+ "tier": "hard",
2854
+ "expected": "standard_research",
2855
+ "predicted": "ethics_review",
2856
+ "correct": false,
2857
+ "valid": true,
2858
+ "error": null,
2859
+ "probs": {
2860
+ "ethics_review": 0.8839,
2861
+ "major_grants": 0.0057,
2862
+ "manual_hold": 0.0015,
2863
+ "standard_research": 0.1089
2864
+ },
2865
+ "latency_s": 0.073
2866
+ },
2867
+ {
2868
+ "id": "hard-sol-a-multi_hop-01",
2869
+ "tier": "hard",
2870
+ "expected": "accept_s2_dusk",
2871
+ "predicted": "accept_s2_dusk",
2872
+ "correct": true,
2873
+ "valid": true,
2874
+ "error": null,
2875
+ "probs": {
2876
+ "accept_s2_dusk": 0.7941,
2877
+ "reject_cutoff": 0.1966,
2878
+ "reject_tier": 0.0054,
2879
+ "reroute_s3": 0.0039
2880
+ },
2881
+ "latency_s": 0.084
2882
+ },
2883
+ {
2884
+ "id": "hard-sol-a-multi_hop-02",
2885
+ "tier": "hard",
2886
+ "expected": "security_primary",
2887
+ "predicted": "executive_bridge",
2888
+ "correct": false,
2889
+ "valid": true,
2890
+ "error": null,
2891
+ "probs": {
2892
+ "executive_bridge": 0.9584,
2893
+ "heron_q2": 0.0153,
2894
+ "insufficient_information": 0.0014,
2895
+ "security_primary": 0.0249
2896
+ },
2897
+ "latency_s": 0.083
2898
+ },
2899
+ {
2900
+ "id": "hard-sol-a-multi_hop-05",
2901
+ "tier": "hard",
2902
+ "expected": "require_director",
2903
+ "predicted": "require_director",
2904
+ "correct": true,
2905
+ "valid": true,
2906
+ "error": null,
2907
+ "probs": {
2908
+ "pay_manager_approved": 0.0131,
2909
+ "reject_category": 0.0213,
2910
+ "require_director": 0.9646,
2911
+ "require_finance_board": 0.001
2912
+ },
2913
+ "latency_s": 0.082
2914
+ },
2915
+ {
2916
+ "id": "hard-sol-a-multi_hop-07",
2917
+ "tier": "hard",
2918
+ "expected": "admit",
2919
+ "predicted": "deny_prerequisite",
2920
+ "correct": false,
2921
+ "valid": true,
2922
+ "error": null,
2923
+ "probs": {
2924
+ "admit": 0.0139,
2925
+ "deny_prerequisite": 0.9698,
2926
+ "deny_score": 0.0107,
2927
+ "manual_equivalency": 0.0056
2928
+ },
2929
+ "latency_s": 0.083
2930
+ },
2931
+ {
2932
+ "id": "hard-sol-a-multi_hop-08",
2933
+ "tier": "hard",
2934
+ "expected": "central_engineering",
2935
+ "predicted": "central_engineering",
2936
+ "correct": true,
2937
+ "valid": true,
2938
+ "error": null,
2939
+ "probs": {
2940
+ "central_engineering": 0.9808,
2941
+ "insufficient_log": 0.002,
2942
+ "local_maintenance": 0.0166,
2943
+ "safety_shutdown": 0.0006
2944
+ },
2945
+ "latency_s": 0.082
2946
+ },
2947
+ {
2948
+ "id": "hard-sol-a-multi_hop-09",
2949
+ "tier": "hard",
2950
+ "expected": "require_steward",
2951
+ "predicted": "require_steward",
2952
+ "correct": true,
2953
+ "valid": true,
2954
+ "error": null,
2955
+ "probs": {
2956
+ "confirm_l3": 0.0001,
2957
+ "reject_capacity": 0.0002,
2958
+ "require_steward": 0.9996,
2959
+ "switch_l4": 0.0001
2960
+ },
2961
+ "latency_s": 0.082
2962
+ },
2963
+ {
2964
+ "id": "hard-sol-a-multi_hop-10",
2965
+ "tier": "hard",
2966
+ "expected": "approve_45_days",
2967
+ "predicted": "approve_45_days",
2968
+ "correct": true,
2969
+ "valid": true,
2970
+ "error": null,
2971
+ "probs": {
2972
+ "approve_45_days": 0.952,
2973
+ "deny_sensitive": 0.013,
2974
+ "needs_field_review": 0.0044,
2975
+ "reduce_to_30_days": 0.0306
2976
+ },
2977
+ "latency_s": 0.082
2978
+ },
2979
+ {
2980
+ "id": "hard-sol-a-multi_hop-12",
2981
+ "tier": "hard",
2982
+ "expected": "needs_authentication",
2983
+ "predicted": "covered_recurrence",
2984
+ "correct": false,
2985
+ "valid": true,
2986
+ "error": null,
2987
+ "probs": {
2988
+ "covered_recurrence": 0.7971,
2989
+ "expired": 0.0286,
2990
+ "needs_authentication": 0.168,
2991
+ "wrong_issue_class": 0.0064
2992
+ },
2993
+ "latency_s": 0.082
2994
+ },
2995
+ {
2996
+ "id": "hard-sol-a-trap-02",
2997
+ "tier": "hard",
2998
+ "expected": "no",
2999
+ "predicted": "no",
3000
+ "correct": true,
3001
+ "valid": true,
3002
+ "error": null,
3003
+ "probs": {
3004
+ "yes": 0.001,
3005
+ "no": 0.999
3006
+ },
3007
+ "latency_s": 0.068
3008
+ },
3009
+ {
3010
+ "id": "hard-sol-a-trap-04",
3011
+ "tier": "hard",
3012
+ "expected": "no",
3013
+ "predicted": "no",
3014
+ "correct": true,
3015
+ "valid": true,
3016
+ "error": null,
3017
+ "probs": {
3018
+ "yes": 0.0138,
3019
+ "no": 0.9862
3020
+ },
3021
+ "latency_s": 0.068
3022
+ },
3023
+ {
3024
+ "id": "hard-sol-a-trap-05",
3025
+ "tier": "hard",
3026
+ "expected": "0",
3027
+ "predicted": "0",
3028
+ "correct": true,
3029
+ "valid": true,
3030
+ "error": null,
3031
+ "probs": {
3032
+ "0": 0.8765,
3033
+ "1": 0.0016,
3034
+ "2": 0.1212,
3035
+ "3": 0.0007
3036
+ },
3037
+ "latency_s": 0.072
3038
+ },
3039
+ {
3040
+ "id": "hard-sol-a-trap-06",
3041
+ "tier": "hard",
3042
+ "expected": "keep_open_unresolved",
3043
+ "predicted": "keep_open_unresolved",
3044
+ "correct": true,
3045
+ "valid": true,
3046
+ "error": null,
3047
+ "probs": {
3048
+ "await_internal_metrics": 0.0001,
3049
+ "close_agent_verified": 0.0001,
3050
+ "close_confirmed": 0.0003,
3051
+ "keep_open_unresolved": 0.9995
3052
+ },
3053
+ "latency_s": 0.068
3054
+ },
3055
+ {
3056
+ "id": "hard-sol-a-trap-08",
3057
+ "tier": "hard",
3058
+ "expected": "late",
3059
+ "predicted": "late",
3060
+ "correct": true,
3061
+ "valid": true,
3062
+ "error": null,
3063
+ "probs": {
3064
+ "canceled": 0.0,
3065
+ "indeterminate": 0.0,
3066
+ "late": 0.9995,
3067
+ "on_time": 0.0005
3068
+ },
3069
+ "latency_s": 0.069
3070
+ },
3071
+ {
3072
+ "id": "hard-sol-a-trap-10",
3073
+ "tier": "hard",
3074
+ "expected": "blocked_second_review",
3075
+ "predicted": "blocked_second_review",
3076
+ "correct": true,
3077
+ "valid": true,
3078
+ "error": null,
3079
+ "probs": {
3080
+ "blocked_failed_test": 0.0002,
3081
+ "blocked_second_review": 0.9991,
3082
+ "ready_one_review": 0.0003,
3083
+ "ready_two_reviews": 0.0004
3084
+ },
3085
+ "latency_s": 0.068
3086
+ },
3087
+ {
3088
+ "id": "hard-sol-a-trap-13",
3089
+ "tier": "hard",
3090
+ "expected": "no",
3091
+ "predicted": "no",
3092
+ "correct": true,
3093
+ "valid": true,
3094
+ "error": null,
3095
+ "probs": {
3096
+ "yes": 0.0057,
3097
+ "no": 0.9943
3098
+ },
3099
+ "latency_s": 0.069
3100
+ },
3101
+ {
3102
+ "id": "hard-sol-a-trap-15",
3103
+ "tier": "hard",
3104
+ "expected": "no",
3105
+ "predicted": "no",
3106
+ "correct": true,
3107
+ "valid": true,
3108
+ "error": null,
3109
+ "probs": {
3110
+ "yes": 0.1823,
3111
+ "no": 0.8177
3112
+ },
3113
+ "latency_s": 0.069
3114
+ },
3115
+ {
3116
+ "id": "hard-sol-b-judge_hard-01",
3117
+ "tier": "hard",
3118
+ "expected": "yes",
3119
+ "predicted": "yes",
3120
+ "correct": true,
3121
+ "valid": true,
3122
+ "error": null,
3123
+ "probs": {
3124
+ "yes": 0.7158,
3125
+ "no": 0.2842
3126
+ },
3127
+ "latency_s": 0.07
3128
+ },
3129
+ {
3130
+ "id": "hard-sol-b-judge_hard-02",
3131
+ "tier": "hard",
3132
+ "expected": "no",
3133
+ "predicted": "yes",
3134
+ "correct": false,
3135
+ "valid": true,
3136
+ "error": null,
3137
+ "probs": {
3138
+ "yes": 0.8925,
3139
+ "no": 0.1075
3140
+ },
3141
+ "latency_s": 0.067
3142
+ },
3143
+ {
3144
+ "id": "hard-sol-b-judge_hard-05",
3145
+ "tier": "hard",
3146
+ "expected": "yes",
3147
+ "predicted": "yes",
3148
+ "correct": true,
3149
+ "valid": true,
3150
+ "error": null,
3151
+ "probs": {
3152
+ "yes": 0.8177,
3153
+ "no": 0.1823
3154
+ },
3155
+ "latency_s": 0.067
3156
+ },
3157
+ {
3158
+ "id": "hard-sol-b-judge_hard-08",
3159
+ "tier": "hard",
3160
+ "expected": "no",
3161
+ "predicted": "yes",
3162
+ "correct": false,
3163
+ "valid": true,
3164
+ "error": null,
3165
+ "probs": {
3166
+ "yes": 0.9269,
3167
+ "no": 0.0731
3168
+ },
3169
+ "latency_s": 0.068
3170
+ },
3171
+ {
3172
+ "id": "hard-sol-b-judge_hard-10",
3173
+ "tier": "hard",
3174
+ "expected": "no",
3175
+ "predicted": "no",
3176
+ "correct": true,
3177
+ "valid": true,
3178
+ "error": null,
3179
+ "probs": {
3180
+ "yes": 0.1362,
3181
+ "no": 0.8638
3182
+ },
3183
+ "latency_s": 0.068
3184
+ },
3185
+ {
3186
+ "id": "hard-sol-b-judge_hard-14",
3187
+ "tier": "hard",
3188
+ "expected": "no",
3189
+ "predicted": "no",
3190
+ "correct": true,
3191
+ "valid": true,
3192
+ "error": null,
3193
+ "probs": {
3194
+ "yes": 0.0091,
3195
+ "no": 0.9909
3196
+ },
3197
+ "latency_s": 0.068
3198
+ },
3199
+ {
3200
+ "id": "hard-sol-b-judge_hard-15",
3201
+ "tier": "hard",
3202
+ "expected": "yes",
3203
+ "predicted": "yes",
3204
+ "correct": true,
3205
+ "valid": true,
3206
+ "error": null,
3207
+ "probs": {
3208
+ "yes": 0.9158,
3209
+ "no": 0.0842
3210
+ },
3211
+ "latency_s": 0.068
3212
+ },
3213
+ {
3214
+ "id": "hard-sol-b-judge_hard-18",
3215
+ "tier": "hard",
3216
+ "expected": "no",
3217
+ "predicted": "yes",
3218
+ "correct": false,
3219
+ "valid": true,
3220
+ "error": null,
3221
+ "probs": {
3222
+ "yes": 0.9757,
3223
+ "no": 0.0243
3224
+ },
3225
+ "latency_s": 0.068
3226
+ },
3227
+ {
3228
+ "id": "hard-sol-b-long_policy-01",
3229
+ "tier": "hard",
3230
+ "expected": "reject_entire_claim",
3231
+ "predicted": "reject_entire_claim",
3232
+ "correct": true,
3233
+ "valid": true,
3234
+ "error": null,
3235
+ "probs": {
3236
+ "approve_all": 0.0009,
3237
+ "approve_lodging_only": 0.0046,
3238
+ "approve_meals_only": 0.0026,
3239
+ "reject_entire_claim": 0.9919
3240
+ },
3241
+ "latency_s": 0.09
3242
+ },
3243
+ {
3244
+ "id": "hard-sol-b-long_policy-02",
3245
+ "tier": "hard",
3246
+ "expected": "deny_current_export",
3247
+ "predicted": "deny_current_export",
3248
+ "correct": true,
3249
+ "valid": true,
3250
+ "error": null,
3251
+ "probs": {
3252
+ "analytics_owner_only": 0.0127,
3253
+ "deny_current_export": 0.6086,
3254
+ "privacy_and_security": 0.0415,
3255
+ "privacy_only": 0.3371
3256
+ },
3257
+ "latency_s": 0.087
3258
+ },
3259
+ {
3260
+ "id": "hard-sol-b-long_policy-05",
3261
+ "tier": "hard",
3262
+ "expected": "delete_profile_omar_april_and_table",
3263
+ "predicted": "delete_nothing",
3264
+ "correct": false,
3265
+ "valid": true,
3266
+ "error": null,
3267
+ "probs": {
3268
+ "delete_everything": 0.0026,
3269
+ "delete_nothing": 0.8642,
3270
+ "delete_only_profile": 0.1185,
3271
+ "delete_profile_omar_april_and_table": 0.0146
3272
+ },
3273
+ "latency_s": 0.085
3274
+ },
3275
+ {
3276
+ "id": "hard-sol-b-long_policy-06",
3277
+ "tier": "hard",
3278
+ "expected": "east_ward",
3279
+ "predicted": "river_arts",
3280
+ "correct": false,
3281
+ "valid": true,
3282
+ "error": null,
3283
+ "probs": {
3284
+ "east_ward": 0.0057,
3285
+ "harbor_neighbors": 0.0018,
3286
+ "no_award": 0.0049,
3287
+ "river_arts": 0.9876
3288
+ },
3289
+ "latency_s": 0.087
3290
+ },
3291
+ {
3292
+ "id": "hard-sol-b-routing_hard-01",
3293
+ "tier": "hard",
3294
+ "expected": "office_file_repair",
3295
+ "predicted": "office_file_repair",
3296
+ "correct": true,
3297
+ "valid": true,
3298
+ "error": null,
3299
+ "probs": {
3300
+ "data_pipeline": 0.0001,
3301
+ "desktop_support": 0.0001,
3302
+ "finance_review": 0.0002,
3303
+ "office_file_repair": 0.9989,
3304
+ "spreadsheet_formula": 0.0007
3305
+ },
3306
+ "latency_s": 0.069
3307
+ },
3308
+ {
3309
+ "id": "hard-sol-b-routing_hard-02",
3310
+ "tier": "hard",
3311
+ "expected": "ledger_reconciliation",
3312
+ "predicted": "ledger_reconciliation",
3313
+ "correct": true,
3314
+ "valid": true,
3315
+ "error": null,
3316
+ "probs": {
3317
+ "customer_support": 0.0003,
3318
+ "fraud_review": 0.0001,
3319
+ "ledger_reconciliation": 0.999,
3320
+ "mobile_app": 0.0001,
3321
+ "payment_disputes": 0.0005
3322
+ },
3323
+ "latency_s": 0.069
3324
+ },
3325
+ {
3326
+ "id": "hard-sol-b-routing_hard-03",
3327
+ "tier": "hard",
3328
+ "expected": "network_dns",
3329
+ "predicted": "network_dns",
3330
+ "correct": true,
3331
+ "valid": true,
3332
+ "error": null,
3333
+ "probs": {
3334
+ "api_platform": 0.0007,
3335
+ "authoritative_dns": 0.0012,
3336
+ "endpoint_devices": 0.0029,
3337
+ "network_dns": 0.9949,
3338
+ "tls_certificates": 0.0003
3339
+ },
3340
+ "latency_s": 0.071
3341
+ },
3342
+ {
3343
+ "id": "hard-sol-b-routing_hard-07",
3344
+ "tier": "hard",
3345
+ "expected": "release_notarization",
3346
+ "predicted": "release_notarization",
3347
+ "correct": true,
3348
+ "valid": true,
3349
+ "error": null,
3350
+ "probs": {
3351
+ "ci_infrastructure": 0.0001,
3352
+ "code_signing": 0.0002,
3353
+ "endpoint_security": 0.0001,
3354
+ "macos_runtime": 0.0001,
3355
+ "release_notarization": 0.9996
3356
+ },
3357
+ "latency_s": 0.069
3358
+ },
3359
+ {
3360
+ "id": "hard-sol-b-routing_hard-09",
3361
+ "tier": "hard",
3362
+ "expected": "source_access",
3363
+ "predicted": "source_access",
3364
+ "correct": true,
3365
+ "valid": true,
3366
+ "error": null,
3367
+ "probs": {
3368
+ "git_client": 0.0002,
3369
+ "identity_login": 0.0001,
3370
+ "repository_admin": 0.0001,
3371
+ "source_access": 0.9993,
3372
+ "vpn_network": 0.0003
3373
+ },
3374
+ "latency_s": 0.069
3375
+ },
3376
+ {
3377
+ "id": "hard-sol-b-temporal_numeric-01",
3378
+ "tier": "hard",
3379
+ "expected": "late_by_under_2h",
3380
+ "predicted": "within_window",
3381
+ "correct": false,
3382
+ "valid": true,
3383
+ "error": null,
3384
+ "probs": {
3385
+ "before_open": 0.2525,
3386
+ "late_by_2h_or_more": 0.0696,
3387
+ "late_by_under_2h": 0.0818,
3388
+ "within_window": 0.5961
3389
+ },
3390
+ "latency_s": 0.07
3391
+ },
3392
+ {
3393
+ "id": "hard-sol-b-temporal_numeric-02",
3394
+ "tier": "hard",
3395
+ "expected": "over_limit",
3396
+ "predicted": "under_100gb_below",
3397
+ "correct": false,
3398
+ "valid": true,
3399
+ "error": null,
3400
+ "probs": {
3401
+ "at_least_100gb_below": 0.2317,
3402
+ "exactly_equal": 0.1679,
3403
+ "over_limit": 0.1092,
3404
+ "under_100gb_below": 0.4913
3405
+ },
3406
+ "latency_s": 0.072
3407
+ },
3408
+ {
3409
+ "id": "hard-sol-b-temporal_numeric-03",
3410
+ "tier": "hard",
3411
+ "expected": "9_credits",
3412
+ "predicted": "9_credits",
3413
+ "correct": true,
3414
+ "valid": true,
3415
+ "error": null,
3416
+ "probs": {
3417
+ "12_credits": 0.1915,
3418
+ "15_credits": 0.0449,
3419
+ "6_credits": 0.172,
3420
+ "9_credits": 0.5915
3421
+ },
3422
+ "latency_s": 0.07
3423
+ },
3424
+ {
3425
+ "id": "hard-sol-b-temporal_numeric-04",
3426
+ "tier": "hard",
3427
+ "expected": "pass",
3428
+ "predicted": "narrow_fail",
3429
+ "correct": false,
3430
+ "valid": true,
3431
+ "error": null,
3432
+ "probs": {
3433
+ "clear_fail": 0.2091,
3434
+ "narrow_fail": 0.6812,
3435
+ "pass": 0.1098
3436
+ },
3437
+ "latency_s": 0.069
3438
+ },
3439
+ {
3440
+ "id": "hard-sol-c-judge_hard-03",
3441
+ "tier": "hard",
3442
+ "expected": "no",
3443
+ "predicted": "yes",
3444
+ "correct": false,
3445
+ "valid": true,
3446
+ "error": null,
3447
+ "probs": {
3448
+ "yes": 0.6666,
3449
+ "no": 0.3334
3450
+ },
3451
+ "latency_s": 0.069
3452
+ },
3453
+ {
3454
+ "id": "hard-sol-c-judge_hard-05",
3455
+ "tier": "hard",
3456
+ "expected": "yes",
3457
+ "predicted": "yes",
3458
+ "correct": true,
3459
+ "valid": true,
3460
+ "error": null,
3461
+ "probs": {
3462
+ "yes": 0.658,
3463
+ "no": 0.342
3464
+ },
3465
+ "latency_s": 0.073
3466
+ },
3467
+ {
3468
+ "id": "hard-sol-c-judge_hard-07",
3469
+ "tier": "hard",
3470
+ "expected": "no",
3471
+ "predicted": "no",
3472
+ "correct": true,
3473
+ "valid": true,
3474
+ "error": null,
3475
+ "probs": {
3476
+ "yes": 0.1605,
3477
+ "no": 0.8395
3478
+ },
3479
+ "latency_s": 0.069
3480
+ },
3481
+ {
3482
+ "id": "hard-sol-c-judge_hard-08",
3483
+ "tier": "hard",
3484
+ "expected": "no",
3485
+ "predicted": "yes",
3486
+ "correct": false,
3487
+ "valid": true,
3488
+ "error": null,
3489
+ "probs": {
3490
+ "yes": 0.956,
3491
+ "no": 0.044
3492
+ },
3493
+ "latency_s": 0.076
3494
+ },
3495
+ {
3496
+ "id": "hard-sol-c-judge_hard-09",
3497
+ "tier": "hard",
3498
+ "expected": "yes",
3499
+ "predicted": "yes",
3500
+ "correct": true,
3501
+ "valid": true,
3502
+ "error": null,
3503
+ "probs": {
3504
+ "yes": 0.8888,
3505
+ "no": 0.1112
3506
+ },
3507
+ "latency_s": 0.073
3508
+ },
3509
+ {
3510
+ "id": "hard-sol-c-judge_hard-10",
3511
+ "tier": "hard",
3512
+ "expected": "yes",
3513
+ "predicted": "yes",
3514
+ "correct": true,
3515
+ "valid": true,
3516
+ "error": null,
3517
+ "probs": {
3518
+ "yes": 0.8496,
3519
+ "no": 0.1504
3520
+ },
3521
+ "latency_s": 0.069
3522
+ },
3523
+ {
3524
+ "id": "hard-sol-c-judge_hard-11",
3525
+ "tier": "hard",
3526
+ "expected": "no",
3527
+ "predicted": "no",
3528
+ "correct": true,
3529
+ "valid": true,
3530
+ "error": null,
3531
+ "probs": {
3532
+ "yes": 0.011,
3533
+ "no": 0.989
3534
+ },
3535
+ "latency_s": 0.069
3536
+ },
3537
+ {
3538
+ "id": "hard-sol-c-judge_hard-13",
3539
+ "tier": "hard",
3540
+ "expected": "no",
3541
+ "predicted": "no",
3542
+ "correct": true,
3543
+ "valid": true,
3544
+ "error": null,
3545
+ "probs": {
3546
+ "yes": 0.3685,
3547
+ "no": 0.6315
3548
+ },
3549
+ "latency_s": 0.07
3550
+ },
3551
+ {
3552
+ "id": "hard-sol-c-judge_hard-15",
3553
+ "tier": "hard",
3554
+ "expected": "yes",
3555
+ "predicted": "yes",
3556
+ "correct": true,
3557
+ "valid": true,
3558
+ "error": null,
3559
+ "probs": {
3560
+ "yes": 0.9757,
3561
+ "no": 0.0243
3562
+ },
3563
+ "latency_s": 0.068
3564
+ },
3565
+ {
3566
+ "id": "hard-sol-c-multi_hop-07",
3567
+ "tier": "hard",
3568
+ "expected": "programme_review",
3569
+ "predicted": "programme_review",
3570
+ "correct": true,
3571
+ "valid": true,
3572
+ "error": null,
3573
+ "probs": {
3574
+ "partial_release": 0.0038,
3575
+ "programme_review": 0.6271,
3576
+ "reject_draw": 0.0025,
3577
+ "release_draw": 0.3666
3578
+ },
3579
+ "latency_s": 0.084
3580
+ },
3581
+ {
3582
+ "id": "hard-sol-c-multi_hop-09",
3583
+ "tier": "hard",
3584
+ "expected": "bus_substitution",
3585
+ "predicted": "pass_fen_cross",
3586
+ "correct": false,
3587
+ "valid": true,
3588
+ "error": null,
3589
+ "probs": {
3590
+ "bus_substitution": 0.0775,
3591
+ "call_fen_cross": 0.0817,
3592
+ "pass_fen_cross": 0.8226,
3593
+ "terminate_oakmere": 0.0182
3594
+ },
3595
+ "latency_s": 0.084
3596
+ },
3597
+ {
3598
+ "id": "hard-sol-c-multi_hop-11",
3599
+ "tier": "hard",
3600
+ "expected": "coverage_excluded",
3601
+ "predicted": "coverage_excluded",
3602
+ "correct": true,
3603
+ "valid": true,
3604
+ "error": null,
3605
+ "probs": {
3606
+ "adjuster_referral": 0.001,
3607
+ "coverage_confirmed": 0.0576,
3608
+ "coverage_excluded": 0.9397,
3609
+ "documentation_needed": 0.0017
3610
+ },
3611
+ "latency_s": 0.087
3612
+ },
3613
+ {
3614
+ "id": "hard-sol-c-multi_hop-12",
3615
+ "tier": "hard",
3616
+ "expected": "conservation_hold",
3617
+ "predicted": "conservation_hold",
3618
+ "correct": true,
3619
+ "valid": true,
3620
+ "error": null,
3621
+ "probs": {
3622
+ "conservation_hold": 0.6161,
3623
+ "deny_request": 0.2105,
3624
+ "digital_copy_only": 0.1446,
3625
+ "supervised_delivery": 0.0289
3626
+ },
3627
+ "latency_s": 0.071
3628
+ },
3629
+ {
3630
+ "id": "hard-sol-c-multi_hop-13",
3631
+ "tier": "hard",
3632
+ "expected": "normal_release",
3633
+ "predicted": "normal_release",
3634
+ "correct": true,
3635
+ "valid": true,
3636
+ "error": null,
3637
+ "probs": {
3638
+ "engineering_review": 0.0,
3639
+ "normal_release": 0.9999,
3640
+ "rework": 0.0,
3641
+ "scrap_lot": 0.0
3642
+ },
3643
+ "latency_s": 0.087
3644
+ }
3645
+ ]
3646
+ }
eval/train.log ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [args] {"model": "Mapika/decider-2b", "data": "/content/data/coreai.pkl", "out": "/content/runs/delta", "epochs": 3.0, "lr": 8e-06, "wd": 0.0, "warmup": 20, "max_tokens": 12288, "accum": 2, "max_ctx": 16384, "brier_w": 0.0, "label_smooth": 0.0, "eval_every": 100000, "eval_limit": 100, "train_cap": 0, "seed": 0, "resample_every_epoch": true, "none_prob": 0.0, "schema_first_prob": 0.0, "max_options": 255}
2
+ [data] train examples 1399; tokenizing...
3
+ [data] 1399 items, 1399 questions, 2.0M tokens, tokenized in 5s
4
+ [sched] 101 optimizer steps/epoch, 303 total
5
+ [train] step 20/303 ep 0 ce 0.6321 gn 51.25 lr 8.00e-06 3.4min eta 49min 1849tok/s mem 20GB
6
+ [train] step 40/303 ep 0 ce 0.5594 gn 76.50 lr 7.90e-06 4.4min eta 13min 7294tok/s mem 20GB
7
+ [train] step 60/303 ep 0 ce 0.5451 gn 29.50 lr 7.61e-06 5.2min eta 9min 8757tok/s mem 20GB
8
+ [train] step 80/303 ep 0 ce 0.6450 gn 50.25 lr 7.15e-06 6.0min eta 9min 8408tok/s mem 20GB
9
+ [train] step 100/303 ep 0 ce 0.5456 gn 43.00 lr 6.52e-06 6.9min eta 8min 9055tok/s mem 20GB
10
+ [train] step 120/303 ep 1 ce 0.2079 gn 147.00 lr 5.78e-06 7.8min eta 8min 7719tok/s mem 20GB
11
+ [train] step 140/303 ep 1 ce 0.1637 gn 70.00 lr 4.95e-06 8.5min eta 6min 8820tok/s mem 20GB
12
+ [train] step 160/303 ep 1 ce 0.1428 gn 25.00 lr 4.07e-06 9.3min eta 6min 8879tok/s mem 20GB
13
+ [train] step 180/303 ep 1 ce 0.2858 gn 19.00 lr 3.18e-06 10.1min eta 5min 8872tok/s mem 20GB
14
+ [train] step 200/303 ep 1 ce 0.2716 gn 71.50 lr 2.34e-06 10.9min eta 4min 8866tok/s mem 20GB
15
+ [train] step 220/303 ep 2 ce 0.0855 gn 3.00 lr 1.58e-06 11.8min eta 4min 7987tok/s mem 20GB
16
+ [train] step 240/303 ep 2 ce 0.0860 gn 0.12 lr 9.39e-07 12.7min eta 3min 8692tok/s mem 20GB
17
+ [train] step 260/303 ep 2 ce 0.1430 gn 117.50 lr 4.47e-07 13.4min eta 2min 8762tok/s mem 20GB
18
+ [train] step 280/303 ep 2 ce 0.1092 gn 14.69 lr 1.30e-07 14.2min eta 1min 8907tok/s mem 20GB
19
+ [train] step 300/303 ep 2 ce 0.1162 gn 61.00 lr 2.22e-09 15.0min eta 0min 8823tok/s mem 20GB
20
+ [save] /content/runs/delta/model
21
+ [eval] coreai_v2_heldout n= 100 acc=0.770 (chance 0.30) nll=0.935 brier=0.385 ece=0.183 aurc=0.086 acc@80=0.825
22
+ [eval-agg] step 303 {"in_task": {"acc": 0.77, "nll": 0.93548583984375, "brier": 0.38499680161476135, "ece": 0.18304700106382374, "aurc": 0.08616249357070915, "acc_at_80": 0.825, "chance": 0.29995238095238097}, "heldout": {}, "n_in": 1, "n_heldout": 0}
23
+ [done] saved /content/runs/delta/model
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "eos_token_id": 248044,
4
+ "transformers_version": "5.16.1",
5
+ "use_cache": true
6
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:493d0d860af7359ca092c9b5fb3e684b31687cdbed9d30c932a6eb4d78513126
3
+ size 3763692048
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523
3
+ size 19989325
tokenizer_config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "audio_bos_token": "<|audio_start|>",
4
+ "audio_eos_token": "<|audio_end|>",
5
+ "audio_token": "<|audio_pad|>",
6
+ "backend": "tokenizers",
7
+ "bos_token": null,
8
+ "clean_up_tokenization_spaces": false,
9
+ "eos_token": "<|endoftext|>",
10
+ "errors": "replace",
11
+ "image_token": "<|image_pad|>",
12
+ "is_local": false,
13
+ "local_files_only": false,
14
+ "model_max_length": 262144,
15
+ "model_specific_special_tokens": {
16
+ "audio_bos_token": "<|audio_start|>",
17
+ "audio_eos_token": "<|audio_end|>",
18
+ "audio_token": "<|audio_pad|>",
19
+ "image_token": "<|image_pad|>",
20
+ "video_token": "<|video_pad|>",
21
+ "vision_bos_token": "<|vision_start|>",
22
+ "vision_eos_token": "<|vision_end|>"
23
+ },
24
+ "pad_token": "<|endoftext|>",
25
+ "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
26
+ "split_special_tokens": false,
27
+ "tokenizer_class": "Qwen2Tokenizer",
28
+ "unk_token": null,
29
+ "video_token": "<|video_pad|>",
30
+ "vision_bos_token": "<|vision_start|>",
31
+ "vision_eos_token": "<|vision_end|>"
32
+ }