Spaces:
Sleeping
Sleeping
zofiasmolenasana
fix(vertex): synchronous CustomJob.submit; train UI GCP arch sync
93e4108 unverified | <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <title>RAG Evaluation Dashboard</title> | |
| <style> | |
| :root{--bg:#f5f6f8;--card:#fff;--border:#e0e0e0;--accent:#2563eb;--accent-light:#dbeafe; | |
| --green:#059669;--green-bg:#ecfdf5;--green-border:#a7f3d0; | |
| --red:#dc2626;--red-bg:#fef2f2;--red-border:#fecaca; | |
| --orange:#d97706;--orange-bg:#fffbeb;--orange-border:#fcd34d; | |
| --gray:#6b7280;--gray-light:#9ca3af;--mono:'SF Mono','Cascadia Code',Menlo,monospace} | |
| *{box-sizing:border-box;margin:0;padding:0} | |
| body{font-family:'Inter','Segoe UI',system-ui,sans-serif;background:var(--bg);color:#1a1a1a;font-size:14px} | |
| /* Top bar */ | |
| .topbar{background:#1e293b;color:#fff;padding:14px 24px;display:flex;align-items:center;gap:16px;position:sticky;top:0;z-index:100} | |
| .topbar h1{font-size:18px;font-weight:700;letter-spacing:-.3px} | |
| .topbar .nav-tabs{display:flex;gap:4px;margin-left:24px} | |
| .topbar .nav-tab{padding:6px 16px;border-radius:6px;cursor:pointer;font-size:13px;font-weight:600;color:#94a3b8;transition:background .15s} | |
| .topbar .nav-tab:hover{background:rgba(255,255,255,.1)} | |
| .topbar .nav-tab.active{background:var(--accent);color:#fff} | |
| .topbar .stats{margin-left:auto;font-size:12px;color:#94a3b8;display:flex;gap:16px} | |
| .topbar .stats span{color:#cbd5e1;font-weight:600} | |
| .container{max-width:1400px;margin:0 auto;padding:20px} | |
| .panel{display:none}.panel.active{display:block} | |
| /* Filters */ | |
| .filters{display:flex;flex-wrap:wrap;gap:10px;margin-bottom:18px;align-items:center;background:var(--card);padding:14px 18px;border-radius:10px;border:1px solid var(--border)} | |
| .filter-group{display:flex;align-items:center;gap:6px} | |
| .filter-group label{font-size:12px;font-weight:600;color:var(--gray)} | |
| .filter-group select,.filter-group input{font-size:13px;padding:5px 10px;border:1px solid var(--border);border-radius:6px;background:#fff} | |
| .filter-group select{min-width:120px} | |
| .filter-group input[type=text]{width:220px} | |
| /* βββ MODEL LEADERBOARD TABLE βββ */ | |
| .lb-card{background:var(--card);border:1px solid var(--border);border-radius:10px;padding:20px;box-shadow:0 1px 3px rgba(0,0,0,.04)} | |
| .lb-card h3{font-size:15px;font-weight:700;margin-bottom:14px;display:flex;align-items:center;gap:8px} | |
| .lb-card h3 .badge{font-size:10px;padding:2px 8px;border-radius:10px;font-weight:600;color:#fff;background:var(--accent)} | |
| table.lb{width:100%;border-collapse:collapse;font-size:13px} | |
| table.lb th{text-align:left;padding:8px 10px;background:#f1f5f9;border-bottom:2px solid var(--border);font-weight:600;font-size:11px;color:var(--gray);text-transform:uppercase;letter-spacing:.3px;cursor:pointer;user-select:none;white-space:nowrap;position:sticky;top:0} | |
| table.lb th:hover{background:#e2e8f0} | |
| table.lb th.sorted-asc::after{content:' β²';font-size:9px} | |
| table.lb th.sorted-desc::after{content:' βΌ';font-size:9px} | |
| table.lb td{padding:7px 10px;border-bottom:1px solid #f0f0f0;white-space:nowrap} | |
| table.lb tr:hover td{background:#f8faff} | |
| table.lb .best-cell{font-weight:700} | |
| .metric-good{color:var(--green);font-weight:600} | |
| .metric-ok{color:var(--orange);font-weight:500} | |
| .metric-bad{color:var(--red)} | |
| .bar-cell{position:relative} | |
| .bar-bg{position:absolute;left:0;top:0;bottom:0;border-radius:0 3px 3px 0;opacity:.12} | |
| .bar-bg-green{background:var(--green)}.bar-bg-orange{background:var(--orange)}.bar-bg-red{background:var(--red)} | |
| /* βββ QUERY ROWS βββ */ | |
| .q-row{background:var(--card);border:1px solid var(--border);border-radius:10px;margin-bottom:8px;overflow:hidden;transition:box-shadow .15s} | |
| .q-row:hover{box-shadow:0 2px 8px rgba(0,0,0,.06)} | |
| .q-header{display:flex;align-items:center;gap:10px;padding:12px 16px;cursor:pointer;user-select:none} | |
| .q-header:hover{background:#fafbff} | |
| .q-num{min-width:30px;font-weight:800;font-size:15px;color:var(--accent)} | |
| .q-text{flex:1;font-size:13px;line-height:1.5} | |
| .q-text b{color:#334155} | |
| .q-expected{font-size:12px;color:var(--green);font-weight:600;max-width:120px;text-align:right;overflow:hidden;text-overflow:ellipsis;white-space:nowrap} | |
| .pill{display:inline-flex;align-items:center;gap:3px;font-size:10px;font-weight:700;padding:2px 8px;border-radius:10px} | |
| .pill-green{background:#d1fae5;color:#065f46} | |
| .pill-red{background:#fee2e2;color:#991b1b} | |
| .pill-orange{background:#fef3c7;color:#92400e} | |
| .pill-blue{background:var(--accent-light);color:#1e40af} | |
| .pill-gray{background:#f1f5f9;color:#475569} | |
| .chevron{font-size:12px;color:var(--gray-light);transition:transform .15s;flex-shrink:0;width:16px;text-align:center} | |
| .q-row.open>.q-header .chevron{transform:rotate(90deg)} | |
| .q-summary{display:flex;gap:4px;align-items:center;flex-shrink:0} | |
| .q-models{display:none;border-top:1px solid #f0f0f0;background:#fafbfc} | |
| .q-row.open>.q-models{display:block} | |
| .m-row{border-bottom:1px solid #f0f0f0} | |
| .m-row:last-child{border-bottom:none} | |
| .m-header{display:flex;align-items:center;gap:10px;padding:10px 16px 10px 46px;cursor:pointer;font-size:13px;user-select:none} | |
| .m-header:hover{background:#f0f4ff} | |
| .m-name{font-weight:600;min-width:220px;color:#334155;overflow:hidden;text-overflow:ellipsis;white-space:nowrap} | |
| .m-answer{flex:1;font-size:12px;color:#475569;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;font-family:var(--mono)} | |
| .m-stats{display:flex;gap:6px;align-items:center;flex-shrink:0} | |
| .m-row.open>.m-header{background:#eef2ff} | |
| .m-chunks{display:none;padding:8px 16px 12px 66px;background:#f5f6fa} | |
| .m-row.open>.m-chunks{display:block} | |
| .chunk{background:#fff;border:1px solid #e5e7eb;border-radius:6px;padding:8px 12px;margin-bottom:6px;font-size:12px} | |
| .chunk-top{display:flex;align-items:center;gap:8px;margin-bottom:4px;flex-wrap:wrap} | |
| .chunk-rank{font-weight:800;color:var(--accent);font-size:13px} | |
| .chunk-pos{font-size:10px;color:var(--gray-light)} | |
| .chunk-text{white-space:pre-wrap;word-break:break-word;font-family:var(--mono);font-size:11px;color:#374151;max-height:160px;overflow-y:auto;background:#f9fafb;padding:6px 8px;border-radius:4px;line-height:1.6} | |
| .jchip{display:inline-flex;align-items:center;gap:3px;padding:3px 8px;border-radius:5px;font-size:11px;font-weight:600} | |
| .jchip-5{background:#d1fae5;color:#065f46}.jchip-4{background:#ecfdf5;color:#047857} | |
| .jchip-3{background:#fef3c7;color:#92400e}.jchip-2{background:#fee2e2;color:#991b1b} | |
| .jchip-1{background:#fef2f2;color:#7f1d1d} | |
| .ans-box{flex:1;min-width:180px;padding:8px 12px;border-radius:6px;font-size:13px;line-height:1.5} | |
| .ans-expected{background:var(--green-bg);border:1px solid var(--green-border)} | |
| .ans-generated{background:#eff6ff;border:1px solid #bfdbfe} | |
| .ans-label{font-size:10px;font-weight:700;color:var(--gray);text-transform:uppercase;letter-spacing:.4px;margin-bottom:3px} | |
| .loading{text-align:center;padding:60px;color:var(--gray)} | |
| .empty{text-align:center;padding:30px;color:var(--gray);font-size:13px} | |
| .struct-disclaimer{font-size:12px;color:#92400e;background:#fffbeb;border:1px solid var(--orange-border);border-radius:8px;padding:12px 14px;margin-bottom:14px;line-height:1.5} | |
| .st-scatter-wrap{background:var(--card);border:1px solid var(--border);border-radius:10px;padding:16px;margin-top:14px} | |
| .st-scatter-wrap svg{width:100%;height:280px;display:block} | |
| .corr-table{font-size:11px} | |
| .corr-table td,.corr-table th{padding:4px 6px;text-align:center;border:1px solid #e5e7eb} | |
| .corr-table th{font-size:10px;color:var(--gray)} | |
| </style> | |
| </head> | |
| <body> | |
| <div class="topbar"> | |
| <h1>RAG Eval Dashboard</h1> | |
| <div class="nav-tabs"> | |
| <div class="nav-tab active" data-panel="models">Model Leaderboard</div> | |
| <div class="nav-tab" data-panel="queries">Per-Query Evaluation</div> | |
| <div class="nav-tab" data-panel="structure">Structure vs RAG</div> | |
| </div> | |
| <div class="stats"> | |
| <div><span id="stat-queries">0</span> queries</div> | |
| <div><span id="stat-models">0</span> models</div> | |
| <div><span id="stat-sheets">0</span> sheets</div> | |
| </div> | |
| </div> | |
| <div class="container"> | |
| <!-- βββββββ TAB 1: MODEL LEADERBOARD βββββββ --> | |
| <div class="panel active" id="panel-models"> | |
| <div class="filters" id="lb-filters" style="margin-bottom:14px"> | |
| <div class="filter-group"><label>Retrieval:</label><select id="lb-retrieval"> | |
| <option value="all">All strategies</option> | |
| </select></div> | |
| <div class="filter-group"><label>Type:</label><select id="lb-type"> | |
| <option value="all">All</option> | |
| <option value="graph">Graph models only</option> | |
| <option value="baseline">Baselines only</option> | |
| </select></div> | |
| </div> | |
| <div class="lb-card"> | |
| <h3>Models & baselines <span class="badge" id="lb-badge">0 methods</span></h3> | |
| <p style="font-size:12px;color:var(--gray);margin:-6px 0 14px;line-height:1.45"> | |
| GNN chunking plus adapters (pandas, etc.). Oracle runs are listed separately below. | |
| </p> | |
| <div style="overflow-x:auto"> | |
| <table class="lb" id="lb-table"> | |
| <thead><tr> | |
| <th data-col="_rank">#</th> | |
| <th data-col="method">Method</th> | |
| <th data-col="n_chunks">Chunks</th> | |
| <th data-col="recall@1">R@1</th> | |
| <th data-col="recall@5">R@5</th> | |
| <th data-col="recall@10">R@10</th> | |
| <th data-col="mrr">MRR</th> | |
| <th data-col="source_recall@1">sR@1</th> | |
| <th data-col="source_recall@5">sR@5</th> | |
| <th data-col="source_recall@10">sR@10</th> | |
| <th data-col="source_mrr">sMRR</th> | |
| <th data-col="judge_score">Judge Avg</th> | |
| <th data-col="judge_binary_acc">Judge β₯4</th> | |
| </tr></thead> | |
| <tbody id="lb-body"></tbody> | |
| </table> | |
| </div> | |
| </div> | |
| <div class="lb-card oracle-card" style="margin-top:18px;border-left:4px solid #14b8a6"> | |
| <h3>Oracle (no GNN, labels from disk) <span class="badge" id="lb-badge-oracle" style="background:#0d9488">0 methods</span></h3> | |
| <p style="font-size:12px;color:var(--gray);margin:-6px 0 14px;line-height:1.45"> | |
| Same <strong>queries</strong> as above; chunkers run on the per-sheet JSON under | |
| <code>rag_eval_cache</code> (else <code>data/labeled</code>) <strong>without</strong> running the GNN. | |
| The default cache from downloading sheets is usually <strong>not</strong> human-labelled for table | |
| structureβso this is <em>not</em> automatically a βgold structureβ ceiling. It only measures | |
| human-quality chunking if you place <strong>labelled</strong> grids there for the same | |
| <code>sheet_id</code>s as your RAG eval (same Q&A file; separate need for structure labels | |
| on those eval sheets). Training labels alone do not help unless exported for those eval IDs. | |
| </p> | |
| <div style="overflow-x:auto"> | |
| <table class="lb" id="lb-table-oracle"> | |
| <thead><tr id="lb-thead-oracle-placeholder"></tr></thead> | |
| <tbody id="lb-body-oracle"></tbody> | |
| </table> | |
| </div> | |
| </div> | |
| </div> | |
| <!-- βββββββ TAB 2: PER-QUERY EVALUATION βββββββ --> | |
| <div class="panel" id="panel-queries"> | |
| <div class="filters"> | |
| <div class="filter-group"><label>Sheet:</label><select id="f-sheet"><option value="all">All Sheets</option></select></div> | |
| <div class="filter-group"><label>Difficulty:</label><select id="f-diff"><option value="all">All</option><option value="1">1 (Easy)</option><option value="2">2</option><option value="3">3</option><option value="4">4</option><option value="5">5 (Hard)</option></select></div> | |
| <div class="filter-group"><label>Result:</label><select id="f-result"><option value="all">All</option><option value="any-hit">Any model found answer</option><option value="all-miss">No model found answer</option><option value="mixed">Some found, some didn't</option></select></div> | |
| <div class="filter-group"><label>Search:</label><input type="text" id="f-search" placeholder="Filter by question text..."></div> | |
| <div class="filter-group"><label>Sort:</label><select id="f-sort"> | |
| <option value="idx">Query order</option> | |
| <option value="best-judge-desc">Best judge (high first)</option> | |
| <option value="best-judge-asc">Best judge (low first)</option> | |
| <option value="avg-judge-desc">Avg judge (high first)</option> | |
| <option value="avg-judge-asc">Avg judge (low first)</option> | |
| <option value="hit-count-desc">Models with answer (most)</option> | |
| <option value="hit-count-asc">Models with answer (fewest)</option> | |
| <option value="difficulty-desc">Difficulty (hard first)</option> | |
| </select></div> | |
| </div> | |
| <div id="query-list"></div> | |
| </div> | |
| <!-- βββββββ TAB 3: STRUCTURE VS RAG βββββββ --> | |
| <div class="panel" id="panel-structure"> | |
| <div class="filters"> | |
| <div class="filter-group"><label>RAG strategy (join key):</label> | |
| <select id="st-strategy"> | |
| <option value="hybrid_rrf">hybrid_rrf</option> | |
| <option value="vector">vector</option> | |
| <option value="bm25">bm25</option> | |
| <option value="hybrid_rrf_rerank">hybrid_rrf_rerank</option> | |
| </select> | |
| </div> | |
| <div class="filter-group"><button type="button" id="st-refresh" style="padding:6px 14px;border-radius:6px;border:1px solid var(--border);background:#fff;cursor:pointer;font-weight:600">Reload</button></div> | |
| </div> | |
| <div id="st-disclaimer" class="struct-disclaimer" style="display:none"></div> | |
| <div class="lb-card"> | |
| <h3>Structure vs gold + RAG (same retrieval strategy)</h3> | |
| <p style="font-size:12px;color:var(--gray);margin:-6px 0 14px;line-height:1.45"> | |
| Rows: GNN chunking runs (<code>graph_*</code> / <code>graph_row_*</code>) with human labels on held-out sheets. | |
| <strong>Inference timing</strong> is wall-clock <code>predict_sheet</code> per sheet (graph build + forward + post); ms/node uses non-empty <code>value</code> cells as graph nodes (same as the GNN). | |
| <strong>Oracle RAG scores</strong> (human chunking) measure a ceiling that still depends on retrieval, generation, and Q&A difficulty β not only perfect structure. | |
| </p> | |
| <div style="overflow-x:auto"> | |
| <table class="lb" id="st-table"> | |
| <thead><tr> | |
| <th>Method</th> | |
| <th>L4 chunk F1</th> | |
| <th>L4 soft F1</th> | |
| <th>L2 macro F1</th> | |
| <th>Judge</th> | |
| <th>R@1</th> | |
| <th>sR@1</th> | |
| <th title="Mean wall ms per graph node (non-empty value cells); predict_sheet">mean ms/node</th> | |
| <th title="95th percentile ms per graph node across sheets">p95 ms/node</th> | |
| <th title="Sum of successful predict_sheet wall times (s)">predict total s</th> | |
| <th title="total_graph_nodes / predict_total_s">nodes/s</th> | |
| </tr></thead> | |
| <tbody id="st-body"></tbody> | |
| </table> | |
| </div> | |
| </div> | |
| <div class="lb-card" style="margin-top:18px"> | |
| <h3>Spearman (structure vs LLM judge)</h3> | |
| <p style="font-size:12px;color:var(--gray);margin:-6px 0 10px">Exploratory only when n is small.</p> | |
| <div style="overflow-x:auto;font-size:12px" id="st-spearman"></div> | |
| </div> | |
| <div class="lb-card" style="margin-top:18px"> | |
| <h3>Pearson between structure aggregates</h3> | |
| <div style="overflow-x:auto" id="st-pearson"></div> | |
| </div> | |
| <div class="st-scatter-wrap"> | |
| <h3 style="font-size:14px;margin-bottom:8px">Scatter: L4 chunk F1 vs judge score</h3> | |
| <svg id="st-scatter" viewBox="0 0 520 260" xmlns="http://www.w3.org/2000/svg"></svg> | |
| </div> | |
| </div> | |
| </div> | |
| <script> | |
| let detailsData = {}; | |
| let aggData = {}; | |
| let methods = []; | |
| let queryIndex = []; | |
| let lbSortCol = 'judge_score'; | |
| let lbSortAsc = false; | |
| function esc(s) { return s ? s.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>') : ''; } | |
| function jclass(s) { return s >= 4.5 ? 5 : s >= 3.5 ? 4 : s >= 2.5 ? 3 : s >= 1.5 ? 2 : 1; } | |
| function pillClass(s) { const c = jclass(s); return c >= 4 ? 'pill-green' : c >= 3 ? 'pill-orange' : 'pill-red'; } | |
| function jchipCls(s) { return 'jchip-' + jclass(s); } | |
| function shortMethod(m) { return m.replace('/hybrid_rrf','').replace('/bm25','').replace('/vector',''); } | |
| function fmtPct(v) { return v != null ? (v * 100).toFixed(1) + '%' : 'β'; } | |
| function fmtF(v, d=3) { return v != null ? v.toFixed(d) : 'β'; } | |
| function metricCls(v, hi, mid) { return v >= hi ? 'metric-good' : v >= mid ? 'metric-ok' : 'metric-bad'; } | |
| function barWidth(v, max) { return max > 0 ? Math.round(v / max * 100) : 0; } | |
| function barColor(v, hi, mid) { return v >= hi ? 'bar-bg-green' : v >= mid ? 'bar-bg-orange' : 'bar-bg-red'; } | |
| // βββββββ TAB NAVIGATION βββββββ | |
| document.querySelectorAll('.nav-tab').forEach(tab => { | |
| tab.addEventListener('click', () => { | |
| document.querySelectorAll('.nav-tab').forEach(t => t.classList.remove('active')); | |
| document.querySelectorAll('.panel').forEach(p => p.classList.remove('active')); | |
| tab.classList.add('active'); | |
| document.getElementById('panel-' + tab.dataset.panel).classList.add('active'); | |
| }); | |
| }); | |
| // βββββββ MODEL LEADERBOARD βββββββ | |
| /** Chunking names that use human labels only (no GNN) β same eval dataset, different chunk builder. */ | |
| const ORACLE_CHUNKING = new Set(['graph_gold', 'graph_row_gold', 'graph_kg_gold']); | |
| function parseMethodKey(key) { | |
| const parts = key.split('/'); | |
| const retrieval = parts.pop(); | |
| const chunking = parts.join('/'); | |
| const isOracle = ORACLE_CHUNKING.has(chunking); | |
| const isGraph = chunking.startsWith('graph_') && !isOracle; | |
| return { chunking, retrieval, full: key, isGraph, isOracle }; | |
| } | |
| function isOracleMethodKey(fullKey) { | |
| return parseMethodKey(fullKey).isOracle; | |
| } | |
| function getLeaderboardRows(oracleOnly) { | |
| const retFilter = document.getElementById('lb-retrieval').value; | |
| const typeFilter = document.getElementById('lb-type').value; | |
| let rows = Object.entries(aggData) | |
| .filter(([k, v]) => !v.error) | |
| .map(([k, v]) => ({ ...parseMethodKey(k), ...v })); | |
| if (oracleOnly) rows = rows.filter(r => r.isOracle); | |
| else rows = rows.filter(r => !r.isOracle); | |
| if (retFilter !== 'all') rows = rows.filter(r => r.retrieval === retFilter); | |
| if (typeFilter === 'graph') rows = rows.filter(r => r.isGraph || (oracleOnly && r.isOracle)); | |
| if (typeFilter === 'baseline') rows = rows.filter(r => !r.isGraph && !r.isOracle); | |
| const col = lbSortCol; | |
| if (col === '_rank') { | |
| // no-op, keep insertion order | |
| } else { | |
| rows.sort((a, b) => { | |
| const va = a[col] ?? -Infinity, vb = b[col] ?? -Infinity; | |
| return lbSortAsc ? va - vb : vb - va; | |
| }); | |
| } | |
| return rows; | |
| } | |
| function renderLeaderboardTable(oracleOnly) { | |
| const rows = getLeaderboardRows(oracleOnly); | |
| const tbodyId = oracleOnly ? 'lb-body-oracle' : 'lb-body'; | |
| const badgeId = oracleOnly ? 'lb-badge-oracle' : 'lb-badge'; | |
| const tableId = oracleOnly ? 'lb-table-oracle' : 'lb-table'; | |
| const tbody = document.getElementById(tbodyId); | |
| const thead = document.querySelector(`#${tableId} thead tr`); | |
| document.getElementById(badgeId).textContent = rows.length + ' methods'; | |
| const emptyMsg = oracleOnly | |
| ? 'No oracle rows in results. Run eval with <code>graph_row_gold</code> / <code>graph_gold</code> etc. For a meaningful structural oracle, use <strong>labelled</strong> JSON for those <code>sheet_id</code>s in <code>rag_eval_cache</code> (raw downloads are usually unlabelled).' | |
| : 'No methods match these filters.'; | |
| if (!rows.length) { | |
| tbody.innerHTML = `<tr><td colspan="24" style="text-align:center;padding:30px;color:var(--gray);line-height:1.5">${emptyMsg}</td></tr>`; | |
| return; | |
| } | |
| const judgeModelKeys = new Set(); | |
| rows.forEach(r => { | |
| Object.keys(r).filter(k => k.startsWith('judge_vertex_google/') || k.startsWith('judge_openai/') || k.startsWith('judge_anthropic/')).forEach(k => judgeModelKeys.add(k)); | |
| }); | |
| const judgeModels = [...judgeModelKeys].sort(); | |
| thead.innerHTML = ` | |
| <th data-col="_rank">#</th> | |
| <th data-col="method">Method</th> | |
| <th data-col="n_chunks">Chunks</th> | |
| <th data-col="mrr">MRR</th> | |
| <th data-col="recall@1">R@1</th> | |
| <th data-col="recall@5">R@5</th> | |
| <th data-col="source_mrr">sMRR</th> | |
| <th data-col="source_recall@1">sR@1</th> | |
| <th data-col="judge_score">Judge Avg</th> | |
| <th data-col="judge_binary_acc">Correct %</th> | |
| ${judgeModels.map(jm => `<th data-col="${jm}" style="font-size:10px">${jm.replace('judge_vertex_google/','').replace('judge_openai/','').replace('judge_anthropic/','')}</th>`).join('')} | |
| `; | |
| thead.querySelectorAll('th[data-col]').forEach(th => { | |
| th.addEventListener('click', () => { | |
| const col = th.dataset.col; | |
| if (col === lbSortCol) { lbSortAsc = !lbSortAsc; } | |
| else { lbSortCol = col; lbSortAsc = false; } | |
| renderLeaderboard(); | |
| }); | |
| }); | |
| const maxJudge = Math.max(...rows.map(r => r.judge_score || 0), 0.01); | |
| const bestByCol = {}; | |
| for (const col of ['recall@1','recall@5','recall@10','mrr','source_recall@1','source_recall@5','source_recall@10','source_mrr','judge_score','judge_binary_acc', ...judgeModels]) { | |
| bestByCol[col] = Math.max(...rows.map(r => r[col] || 0)); | |
| } | |
| tbody.innerHTML = rows.map((r, i) => { | |
| const isBest = (col) => r[col] != null && r[col] === bestByCol[col] && r[col] > 0; | |
| const bc = (col) => isBest(col) ? ' best-cell' : ''; | |
| const jw = barWidth(r.judge_score || 0, maxJudge); | |
| const sw = barWidth(r['source_recall@1'] || 0, 1); | |
| const perJudgeCells = judgeModels.map(jm => { | |
| const v = r[jm]; | |
| return `<td class="${metricCls(v||0,3.0,2.5)}${bc(jm)}" style="font-size:12px">${v != null ? v.toFixed(2) : 'β'}</td>`; | |
| }).join(''); | |
| const methodBadge = r.isOracle | |
| ? ' <span class="pill" style="font-size:9px;background:#ccfbf1;color:#0f766e;border:1px solid #5eead4">Oracle</span>' | |
| : (r.isGraph ? ' <span class="pill pill-blue" style="font-size:9px">GNN</span>' : ''); | |
| return `<tr> | |
| <td style="color:var(--gray);font-weight:600">${i + 1}</td> | |
| <td><strong>${esc(shortMethod(r.full))}</strong>${methodBadge}</td> | |
| <td>${r.n_chunks || 0}</td> | |
| <td class="${metricCls(r.mrr||0,.28,.20)}${bc('mrr')}">${fmtF(r.mrr)}</td> | |
| <td class="${metricCls(r['recall@1']||0,.25,.15)}${bc('recall@1')}">${fmtPct(r['recall@1'])}</td> | |
| <td class="${metricCls(r['recall@5']||0,.30,.20)}${bc('recall@5')}">${fmtPct(r['recall@5'])}</td> | |
| <td class="${metricCls(r.source_mrr||0,.55,.40)}${bc('source_mrr')}">${fmtF(r.source_mrr)}</td> | |
| <td class="bar-cell${bc('source_recall@1')}"><div class="bar-bg ${barColor(r['source_recall@1']||0,.55,.40)}" style="width:${sw}%"></div>${fmtPct(r['source_recall@1'])}</td> | |
| <td class="bar-cell ${metricCls(r.judge_score||0,3.0,2.5)}${bc('judge_score')}"><div class="bar-bg ${barColor(r.judge_score||0,3.0,2.5)}" style="width:${jw}%"></div><strong>${fmtF(r.judge_score, 2)}</strong></td> | |
| <td class="${metricCls(r.judge_binary_acc||0,.45,.30)}${bc('judge_binary_acc')}">${fmtPct(r.judge_binary_acc)}</td> | |
| ${perJudgeCells} | |
| </tr>`; | |
| }).join(''); | |
| document.querySelectorAll(`#${tableId} th`).forEach(th => { | |
| th.classList.remove('sorted-asc', 'sorted-desc'); | |
| if (th.dataset.col === lbSortCol) { | |
| th.classList.add(lbSortAsc ? 'sorted-asc' : 'sorted-desc'); | |
| } | |
| }); | |
| } | |
| function renderLeaderboard() { | |
| renderLeaderboardTable(false); | |
| renderLeaderboardTable(true); | |
| } | |
| // Sort handlers are attached dynamically in renderLeaderboard() | |
| ['lb-retrieval', 'lb-type'].forEach(id => | |
| document.getElementById(id).addEventListener('change', renderLeaderboard)); | |
| // βββββββ PER-QUERY INDEX βββββββ | |
| function buildQueryIndex() { | |
| if (!methods.length) return; | |
| const refMethod = methods[0]; | |
| const refDetails = detailsData[refMethod] || []; | |
| queryIndex = refDetails.map((d, qi) => { | |
| const perMethod = {}; | |
| let bestJudge = 0, worstJudge = 5, sumJudge = 0, countJudge = 0; | |
| let hitCount = 0, srcHitCount = 0; | |
| for (const m of methods) { | |
| const md = (detailsData[m] || [])[qi]; | |
| if (!md) continue; | |
| perMethod[m] = md; | |
| const js = md.judge_verdict?.score || 0; | |
| if (js > bestJudge) bestJudge = js; | |
| if (js < worstJudge) worstJudge = js; | |
| sumJudge += js; countJudge++; | |
| if (md.answer_hit_rank !== null && md.answer_hit_rank !== undefined) hitCount++; | |
| if (md.source_hit_rank !== null && md.source_hit_rank !== undefined) srcHitCount++; | |
| } | |
| return { | |
| idx: qi, query_id: d.query_id, question: d.question, | |
| expected_answer: d.expected_answer, sheet_id: d.sheet_id, | |
| query_difficulty: d.query_difficulty, sheet_complexity: d.sheet_complexity, | |
| bestJudge, worstJudge, | |
| avgJudge: countJudge ? sumJudge / countJudge : 0, | |
| hitCount, srcHitCount, totalModels: countJudge, perMethod, | |
| }; | |
| }); | |
| } | |
| function filterAndSort() { | |
| const sheet = document.getElementById('f-sheet').value; | |
| const diff = document.getElementById('f-diff').value; | |
| const result = document.getElementById('f-result').value; | |
| const search = document.getElementById('f-search').value.toLowerCase(); | |
| const sort = document.getElementById('f-sort').value; | |
| let qs = queryIndex.filter(q => { | |
| if (sheet !== 'all' && q.sheet_id !== sheet) return false; | |
| if (diff !== 'all' && String(q.query_difficulty) !== diff) return false; | |
| if (search && !q.question.toLowerCase().includes(search)) return false; | |
| if (result === 'any-hit' && q.hitCount === 0) return false; | |
| if (result === 'all-miss' && q.hitCount > 0) return false; | |
| if (result === 'mixed' && (q.hitCount === 0 || q.hitCount === q.totalModels)) return false; | |
| return true; | |
| }); | |
| const sf = { | |
| 'idx': (a, b) => a.idx - b.idx, | |
| 'best-judge-desc': (a, b) => b.bestJudge - a.bestJudge, | |
| 'best-judge-asc': (a, b) => a.bestJudge - b.bestJudge, | |
| 'avg-judge-desc': (a, b) => b.avgJudge - a.avgJudge, | |
| 'avg-judge-asc': (a, b) => a.avgJudge - b.avgJudge, | |
| 'hit-count-desc': (a, b) => b.hitCount - a.hitCount, | |
| 'hit-count-asc': (a, b) => a.hitCount - b.hitCount, | |
| 'difficulty-desc': (a, b) => (b.query_difficulty || 0) - (a.query_difficulty || 0), | |
| }; | |
| qs.sort(sf[sort] || sf['idx']); | |
| return qs; | |
| } | |
| function renderQueryList() { | |
| const qs = filterAndSort(); | |
| const container = document.getElementById('query-list'); | |
| if (!qs.length) { | |
| container.innerHTML = '<div class="empty">No queries match the current filters.</div>'; | |
| return; | |
| } | |
| container.innerHTML = qs.map(q => { | |
| const hitPill = q.hitCount > 0 | |
| ? `<span class="pill pill-green">${q.hitCount}/${q.totalModels} found</span>` | |
| : `<span class="pill pill-red">0/${q.totalModels} found</span>`; | |
| const avgPill = `<span class="pill ${pillClass(q.avgJudge)}">avg ${q.avgJudge.toFixed(1)}</span>`; | |
| const bestPill = `<span class="pill ${pillClass(q.bestJudge)}">best ${q.bestJudge.toFixed(1)}</span>`; | |
| const diffPill = q.query_difficulty ? `<span class="pill pill-gray">diff ${q.query_difficulty}</span>` : ''; | |
| const sortedMethods = methods.slice().sort((a, b) => { | |
| const sa = q.perMethod[a]?.judge_verdict?.score || 0; | |
| const sb = q.perMethod[b]?.judge_verdict?.score || 0; | |
| return sb - sa; | |
| }); | |
| const modelRowsHtml = sortedMethods.map((m, mi) => { | |
| const md = q.perMethod[m]; | |
| if (!md) return ''; | |
| const js = md.judge_verdict?.score || 0; | |
| const pm = md.judge_verdict?.per_model || {}; | |
| const gen = md.generated_answer || 'NOT FOUND'; | |
| const hitRank = md.answer_hit_rank; | |
| const srcRank = md.source_hit_rank; | |
| const hitTag = hitRank != null | |
| ? `<span class="pill pill-green">answer @${hitRank}</span>` | |
| : `<span class="pill pill-red">no answer match</span>`; | |
| const srcTag = srcRank != null | |
| ? `<span class="pill pill-green">source @${srcRank}</span>` | |
| : `<span class="pill pill-red">no source</span>`; | |
| const judgeChips = Object.entries(pm).map(([model, score]) => | |
| `<span class="jchip ${jchipCls(score)}">${model.split('/').pop()}: ${score}/5</span>` | |
| ).join(' '); | |
| const chunks = md.retrieved_chunks || []; | |
| const chunksHtml = chunks.map(c => { | |
| const tags = []; | |
| if (c.contains_answer) tags.push('<span class="pill pill-green">ANSWER MATCH</span>'); | |
| if (c.contains_source) tags.push('<span class="pill pill-blue">SOURCE CELL</span>'); | |
| if (!c.contains_answer && !c.contains_source) tags.push('<span class="pill pill-red">no match</span>'); | |
| const pos = (c.row != null) ? `<span class="chunk-pos">row ${c.row}${c.col != null ? ', col ' + c.col : ''}</span>` : ''; | |
| return `<div class="chunk"> | |
| <div class="chunk-top"><span class="chunk-rank">#${c.rank}</span>${tags.join('')}${pos}</div> | |
| <div class="chunk-text">${esc(c.text)}</div> | |
| </div>`; | |
| }).join(''); | |
| const oracleTag = isOracleMethodKey(m) | |
| ? '<span class="pill" style="font-size:8px;background:#ccfbf1;color:#0f766e;margin-right:4px">Oracle</span>' | |
| : ''; | |
| return `<div class="m-row" id="mr-${q.idx}-${mi}"> | |
| <div class="m-header" onclick="toggleMRow('mr-${q.idx}-${mi}')"> | |
| <span class="chevron">▶</span> | |
| <span class="m-name">${oracleTag}${esc(shortMethod(m))}</span> | |
| <span class="m-answer">${esc(gen.length > 80 ? gen.slice(0,80)+'...' : gen)}</span> | |
| <div class="m-stats">${hitTag}${srcTag}<span class="jchip ${jchipCls(js)}"><b>${js.toFixed(1)}</b>/5</span></div> | |
| <span class="chevron">▶</span> | |
| </div> | |
| <div class="m-chunks"> | |
| <div style="padding:6px 0 10px;display:flex;gap:12px;flex-wrap:wrap;align-items:center"> | |
| <div class="ans-box ans-expected" style="flex:0 1 auto;padding:6px 10px"><div class="ans-label">Expected</div>${esc(q.expected_answer)}</div> | |
| <div class="ans-box ans-generated" style="flex:1;padding:6px 10px"><div class="ans-label">Generated by ${esc(shortMethod(m))}</div>${esc(gen)}</div> | |
| <div style="display:flex;gap:4px;flex-wrap:wrap">${judgeChips}</div> | |
| </div> | |
| <div style="font-size:12px;font-weight:700;color:#334155;margin:6px 0 6px">Retrieved Chunks (${chunks.length})</div> | |
| ${chunksHtml || '<div class="empty">No chunks retrieved</div>'} | |
| </div> | |
| </div>`; | |
| }).join(''); | |
| return `<div class="q-row" id="qr-${q.idx}"> | |
| <div class="q-header" onclick="toggleQRow('qr-${q.idx}')"> | |
| <span class="q-num">${q.idx + 1}</span> | |
| <span class="q-text"><b>Q:</b> ${esc(q.question)}</span> | |
| <span class="q-expected" title="${esc(q.expected_answer)}">${esc(q.expected_answer)}</span> | |
| <div class="q-summary">${hitPill}${avgPill}${bestPill}${diffPill}</div> | |
| <span class="chevron">▶</span> | |
| </div> | |
| <div class="q-models">${modelRowsHtml}</div> | |
| </div>`; | |
| }).join(''); | |
| } | |
| function toggleQRow(id) { document.getElementById(id)?.classList.toggle('open'); } | |
| function toggleMRow(id) { document.getElementById(id)?.classList.toggle('open'); } | |
| // βββββββ STRUCTURE VS RAG βββββββ | |
| let structureAnalysisCache = null; | |
| function renderScatter(rows) { | |
| const svg = document.getElementById('st-scatter'); | |
| if (!svg || !rows.length) { if (svg) svg.innerHTML = ''; return; } | |
| const xs = rows.map(r => r.level4_chunk_f1).filter(v => v != null); | |
| const ys = rows.map(r => r.rag_judge_score).filter(v => v != null); | |
| if (!xs.length || !ys.length) { svg.innerHTML = ''; return; } | |
| const pad = 40, W = 520, H = 260; | |
| const minX = Math.min(...xs), maxX = Math.max(...xs); | |
| const minY = Math.min(...ys), maxY = Math.max(...ys); | |
| const rx = (v) => pad + (maxX > minX ? (v - minX) / (maxX - minX) : 0.5) * (W - 2 * pad); | |
| const ry = (v) => H - pad - (maxY > minY ? (v - minY) / (maxY - minY) : 0.5) * (H - 2 * pad); | |
| let h = `<line x1="${pad}" y1="${H-pad}" x2="${W-pad}" y2="${H-pad}" stroke="#cbd5e1" stroke-width="1"/>`; | |
| h += `<line x1="${pad}" y1="${H-pad}" x2="${pad}" y2="${pad}" stroke="#cbd5e1" stroke-width="1"/>`; | |
| h += `<text x="${W/2}" y="${H-8}" text-anchor="middle" font-size="10" fill="#64748b">L4 chunk F1</text>`; | |
| h += `<text x="12" y="${H/2}" font-size="10" fill="#64748b" transform="rotate(-90 12 ${H/2})">Judge</text>`; | |
| rows.forEach(r => { | |
| if (r.level4_chunk_f1 == null || r.rag_judge_score == null) return; | |
| const cx = rx(r.level4_chunk_f1), cy = ry(r.rag_judge_score); | |
| h += `<circle cx="${cx}" cy="${cy}" r="5" fill="#2563eb" fill-opacity="0.65"><title>${esc(r.method_base)}</title></circle>`; | |
| }); | |
| svg.innerHTML = h; | |
| } | |
| async function loadStructureAnalysis() { | |
| const strat = document.getElementById('st-strategy')?.value || 'hybrid_rrf'; | |
| const disc = document.getElementById('st-disclaimer'); | |
| const tbody = document.getElementById('st-body'); | |
| const spDiv = document.getElementById('st-spearman'); | |
| const pDiv = document.getElementById('st-pearson'); | |
| if (!tbody) return; | |
| tbody.innerHTML = '<tr><td colspan="11">Loadingβ¦</td></tr>'; | |
| try { | |
| const r = await fetch('/api/rag_eval/structure_analysis?strategy=' + encodeURIComponent(strat)).then(x => x.json()); | |
| structureAnalysisCache = r; | |
| if (!r.exists || !r.joined_rows) { | |
| tbody.innerHTML = '<tr><td colspan="11" class="empty">No joined data. Run RAG eval with graph methods and ensure rag_eval_structure_metrics.json exists.</td></tr>'; | |
| if (disc) { disc.style.display = 'none'; } | |
| if (spDiv) spDiv.innerHTML = ''; | |
| if (pDiv) pDiv.innerHTML = ''; | |
| document.getElementById('st-scatter').innerHTML = ''; | |
| return; | |
| } | |
| const an = r.analysis || {}; | |
| if (disc && an.disclaimer) { | |
| disc.textContent = an.disclaimer; | |
| disc.style.display = 'block'; | |
| } | |
| tbody.innerHTML = r.joined_rows.map(row => { | |
| return `<tr> | |
| <td style="font-family:var(--mono);font-size:12px">${esc(row.method_base)}</td> | |
| <td>${fmtF(row.level4_chunk_f1, 3)}</td> | |
| <td>${fmtF(row.level4_soft_chunk_f1, 3)}</td> | |
| <td>${fmtF(row.level2_macro_f1, 3)}</td> | |
| <td>${fmtF(row.rag_judge_score, 2)}</td> | |
| <td>${fmtF(row['rag_recall@1'], 3)}</td> | |
| <td>${fmtF(row['rag_source_recall@1'], 3)}</td> | |
| <td>${row.inf_mean_ms_per_graph_node != null ? fmtF(row.inf_mean_ms_per_graph_node, 2) : 'β'}</td> | |
| <td>${row.inf_p95_ms_per_graph_node != null ? fmtF(row.inf_p95_ms_per_graph_node, 2) : 'β'}</td> | |
| <td>${row.inf_predict_total_s != null ? fmtF(row.inf_predict_total_s, 3) : 'β'}</td> | |
| <td>${row.inf_throughput_nodes_per_s != null ? fmtF(row.inf_throughput_nodes_per_s, 1) : 'β'}</td> | |
| </tr>`; | |
| }).join(''); | |
| const sv = an.spearman_structure_vs_rag_judge || {}; | |
| const spRows = Object.keys(sv).filter(k => sv[k] && sv[k].rho != null).map(k => | |
| `<tr><td style="text-align:left">${esc(k)}</td><td>${fmtF(sv[k].rho, 3)}</td><td>${sv[k].pvalue != null ? fmtF(sv[k].pvalue, 4) : 'β'}</td><td>${sv[k].n}</td></tr>` | |
| ); | |
| if (spDiv) spDiv.innerHTML = spRows.length | |
| ? '<table class="lb"><thead><tr><th>Metric</th><th>Ο</th><th>p</th><th>n</th></tr></thead><tbody>' + spRows + '</tbody></table>' | |
| : '<span class="empty">Not enough overlapping points.</span>'; | |
| const pm = an.structure_metric_pearson || {}; | |
| if (pDiv && pm.matrix && pm.matrix.length) { | |
| const cols = pm.columns || []; | |
| let ht = '<table class="corr-table"><thead><tr><th></th>'; | |
| cols.forEach(c => { ht += '<th title="' + esc(c) + '">' + esc(c.replace('level4_','').replace('level','L')) + '</th>'; }); | |
| ht += '</tr></thead><tbody>'; | |
| pm.matrix.forEach((row, i) => { | |
| ht += '<tr><th>' + esc((cols[i]||'').replace('level4_','').slice(0,8)) + '</th>'; | |
| row.forEach(cell => { | |
| const v = cell == null ? '' : cell; | |
| const bg = v === '' ? '#f9fafb' : `rgba(37,99,235,${Math.min(0.15 + Math.abs(v) * 0.35, 0.85)})`; | |
| ht += '<td style="background:' + bg + '">' + (v === '' ? 'β' : fmtF(v, 2)) + '</td>'; | |
| }); | |
| ht += '</tr>'; | |
| }); | |
| ht += '</tbody></table>'; | |
| pDiv.innerHTML = ht; | |
| } else if (pDiv) pDiv.innerHTML = '<span class="empty">Need β₯2 rows with all structure metrics.</span>'; | |
| renderScatter(r.joined_rows); | |
| } catch (e) { | |
| tbody.innerHTML = '<tr><td colspan="11">Error: ' + esc(String(e)) + '</td></tr>'; | |
| } | |
| } | |
| document.getElementById('st-refresh')?.addEventListener('click', loadStructureAnalysis); | |
| document.getElementById('st-strategy')?.addEventListener('change', loadStructureAnalysis); | |
| // βββββββ LOAD βββββββ | |
| async function loadData() { | |
| document.getElementById('query-list').innerHTML = '<div class="loading">Loading...</div>'; | |
| const [aggResp, detResp] = await Promise.all([ | |
| fetch('/api/rag_eval/results').then(r => r.json()).catch(() => ({})), | |
| fetch('/api/rag_eval/details').then(r => r.json()).catch(() => ({})), | |
| ]); | |
| aggData = aggResp.results || {}; | |
| detailsData = detResp.details || {}; | |
| methods = Object.keys(detailsData).sort((a, b) => { | |
| const ao = isOracleMethodKey(a) ? 0 : 1; | |
| const bo = isOracleMethodKey(b) ? 0 : 1; | |
| if (ao !== bo) return ao - bo; | |
| return a.localeCompare(b); | |
| }); | |
| buildQueryIndex(); | |
| // Populate retrieval filter from actual data | |
| const retrievals = [...new Set(Object.keys(aggData).map(k => k.split('/').pop()))].sort(); | |
| const retSel = document.getElementById('lb-retrieval'); | |
| retSel.innerHTML = '<option value="all">All strategies</option>' + | |
| retrievals.map(r => `<option value="${r}">${r}</option>`).join(''); | |
| if (retrievals.includes('hybrid_rrf')) retSel.value = 'hybrid_rrf'; | |
| else if (retrievals.length === 1) retSel.value = retrievals[0]; | |
| const sheets = [...new Set(queryIndex.map(q => q.sheet_id).filter(Boolean))].sort(); | |
| document.getElementById('f-sheet').innerHTML = '<option value="all">All Sheets</option>' + | |
| sheets.map(s => `<option value="${s}">${s}</option>`).join(''); | |
| document.getElementById('stat-queries').textContent = queryIndex.length; | |
| document.getElementById('stat-models').textContent = Object.keys(aggData).filter(k => !aggData[k]?.error).length; | |
| document.getElementById('stat-sheets').textContent = sheets.length; | |
| renderLeaderboard(); | |
| renderQueryList(); | |
| loadStructureAnalysis(); | |
| } | |
| ['f-sheet','f-diff','f-result','f-sort'].forEach(id => | |
| document.getElementById(id).addEventListener('change', renderQueryList)); | |
| document.getElementById('f-search').addEventListener('input', renderQueryList); | |
| loadData(); | |
| </script> | |
| </body> | |
| </html> | |