Create app.py

app.py

@@ -0,0 +1,475 @@
+"""
+HF Dataset Explorer — view ANY Hugging Face dataset, even when the
+official dataset-viewer is unavailable / stuck / not-yet-built.
+
+Run:  python app.py         (add --share for a public link)
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import tempfile
+import traceback
+
+import gradio as gr
+import pandas as pd
+
+import backend as B
+
+STATE_DEFAULT = {"dataset": "", "config": "", "split": "", "df": None,
+                 "offset": 0, "total": None, "token": ""}
+
+EXAMPLES = [
+    "stanfordnlp/imdb",
+    "openai/gsm8k",
+    "cais/mmlu",
+    "HuggingFaceFW/fineweb",
+    "lmms-lab/COCO-Caption",
+]
+
+
+# --------------------------------------------------------------------------- #
+# tab 1: connect
+# --------------------------------------------------------------------------- #
+def connect(raw_id, token):
+    try:
+        ds = B.normalize_id(raw_id)
+    except B.LoadError as e:
+        return (gr.update(), gr.update(), f"### ❌ {e}", "", gr.update(choices=[]), {})
+
+    lines = [f"## `{ds}`", ""]
+    try:
+        info = B.repo_info(ds, token)
+        lines += [
+            f"- **Downloads (30d):** {info.get('downloads', '?'):,}  ·  **Likes:** {info.get('likes', '?')}",
+            f"- **Modified:** {str(info.get('lastModified', ''))[:19]}",
+            f"- **Tags:** {', '.join(info.get('tags', [])[:14]) or '—'}",
+            f"- **Gated:** {info.get('gated', False)}  ·  **Private:** {info.get('private', False)}",
+        ]
+    except Exception as e:  # noqa: BLE001
+        lines.append(f"> ⚠️ repo metadata unavailable: {e}")
+
+    ok, detail = B.viewer_status(ds, token)
+    lines += ["",
+              f"- **Official viewer:** {'✅ ready' if ok else '⛔ NOT ready'} — `{detail}`",
+              f"  {'' if ok else '→ no problem, this app falls back to parquet / streaming / raw files.'}"]
+
+    try:
+        mapping = B.discover_splits(ds, token)
+    except Exception as e:  # noqa: BLE001
+        mapping = {"default": ["train"]}
+        lines.append(f"> ⚠️ split discovery fell back to guesses: {e}")
+
+    configs = list(mapping)
+    first_cfg = configs[0]
+    splits = mapping[first_cfg]
+    lines.append(f"- **Configs:** {len(configs)} · **Splits in `{first_cfg}`:** {', '.join(splits)}")
+
+    # file tree
+    try:
+        files = B.list_files(ds, token)
+        tab = pd.DataFrame([{"path": f["path"],
+                             "size": f.get("size") or (f.get("lfs") or {}).get("size") or 0}
+                            for f in files if f.get("type") == "file"])
+        if not tab.empty:
+            tab["size"] = tab["size"].map(lambda n: f"{n/1e6:,.2f} MB" if n else "—")
+        tabular = [f["path"] for f in files
+                   if f.get("type") == "file" and f["path"].lower().endswith(B.TABULAR_EXT)]
+    except Exception:
+        tab, tabular = pd.DataFrame(), []
+
+    return (gr.update(choices=configs, value=first_cfg),
+            gr.update(choices=splits, value=splits[0] if splits else None),
+            "\n".join(lines),
+            B.readme(ds, token),
+            gr.update(choices=tabular[:500], value=tabular[0] if tabular else None),
+            {"dataset": ds, "mapping": mapping, "files": tab.to_dict("records")})
+
+
+def on_config_change(cfg, meta):
+    splits = (meta or {}).get("mapping", {}).get(cfg, ["train"])
+    return gr.update(choices=splits, value=splits[0] if splits else None)
+
+
+def file_table(meta):
+    recs = (meta or {}).get("files", [])
+    return pd.DataFrame(recs) if recs else pd.DataFrame({"path": [], "size": []})
+
+
+# --------------------------------------------------------------------------- #
+# tab 2: rows
+# --------------------------------------------------------------------------- #
+def _render(res: B.LoadResult, state, maxlen, cols_pick=None):
+    df = res.df
+    state = dict(state or {})
+    state.update(df=df, offset=res.offset, total=res.total_rows,
+                 strategy=res.strategy)
+
+    media = B.detect_media_columns(df)
+    total = f"{res.total_rows:,}" if res.total_rows else "unknown"
+    banner = (f"**Strategy:** `{res.strategy}` · **rows {res.offset:,}–{res.offset+len(df)-1:,}** "
+              f"of {total} · **{len(df.columns)} columns**"
+              + (f"\n\n_{' · '.join(res.notes)}_" if res.notes else "")
+              + (f"\n\n🖼️ image cols: `{media['image']}`" if media["image"] else "")
+              + (f" 🔊 audio cols: `{media['audio']}`" if media["audio"] else ""))
+
+    keep = [c for c in (cols_pick or []) if c in df.columns]
+    show = df[keep] if keep else df
+    return (B.display_frame(show, maxlen), banner, state,
+            gr.update(choices=[str(c) for c in df.columns],
+                      value=[str(c) for c in df.columns]),
+            gr.update(maximum=max(1, len(df) - 1), value=0))
+
+
+def load_rows(dataset_meta, cfg, split, offset, limit, token, strategy, maxlen, state):
+    ds = (dataset_meta or {}).get("dataset", "")
+    if not ds:
+        return (pd.DataFrame(), "### ❌ Connect to a dataset first (tab 1).", state,
+                gr.update(), gr.update())
+    try:
+        res = B.load_rows(ds, cfg, split, int(offset), int(limit), token, strategy)
+    except Exception as e:  # noqa: BLE001
+        return (pd.DataFrame(), f"### ❌ Could not load rows\n```\n{e}\n```", state,
+                gr.update(), gr.update())
+    state = dict(state or {}); state["dataset"] = ds
+    return _render(res, state, int(maxlen))
+
+
+def page(delta, dataset_meta, cfg, split, offset, limit, token, strategy, maxlen, state):
+    new_off = max(0, int(offset) + delta * int(limit))
+    out = load_rows(dataset_meta, cfg, split, new_off, limit, token, strategy, maxlen, state)
+    return (*out, new_off)
+
+
+def _state_df(state) -> pd.DataFrame:
+    df = (state or {}).get("df")
+    return df if isinstance(df, pd.DataFrame) else pd.DataFrame()
+
+
+def apply_filter(query, regex_col, regex, state, maxlen):
+    df = _state_df(state)
+    if df.empty:
+        return pd.DataFrame(), "No data loaded."
+    note = []
+    try:
+        if query.strip():
+            df = df.query(query, engine="python")
+            note.append(f"query `{query}`")
+        if regex.strip() and regex_col:
+            df = df[df[regex_col].astype(str).str.contains(regex, case=False, regex=True, na=False)]
+            note.append(f"regex `{regex}` on `{regex_col}`")
+    except Exception as e:  # noqa: BLE001
+        return pd.DataFrame(), f"❌ filter error: `{e}`"
+    return B.display_frame(df, int(maxlen)), f"**{len(df):,} matching rows** ({' + '.join(note) or 'no filter'})"
+
+
+def pick_columns(cols, state, maxlen):
+    df = _state_df(state)
+    if df.empty:
+        return pd.DataFrame()
+    keep = [c for c in df.columns if str(c) in set(cols or [])]
+    return B.display_frame(df[keep] if keep else df, int(maxlen))
+
+
+# --------------------------------------------------------------------------- #
+# tab 3: record inspector
+# --------------------------------------------------------------------------- #
+def inspect(idx, state):
+    df = _state_df(state)
+    if df.empty:
+        return {}, "No data loaded.", None, None
+    i = max(0, min(int(idx), len(df) - 1))
+    row = df.iloc[i]
+    payload = {str(k): B.json_safe(v) for k, v in row.items()}
+
+    md = [f"### Row {i} (absolute #{(state or {}).get('offset', 0) + i})"]
+    for k, v in row.items():
+        s = B.stringify(v, 4000)
+        md.append(f"**{k}**\n\n```\n{s}\n```")
+
+    img = aud = None
+    media = B.detect_media_columns(df)
+    if media["image"]:
+        img = B.media_to_display(row[media["image"][0]])
+    if media["audio"]:
+        aud = B.media_to_display(row[media["audio"][0]])
+    return payload, "\n\n".join(md), img, aud
+
+
+# --------------------------------------------------------------------------- #
+# tab 4: stats
+# --------------------------------------------------------------------------- #
+def profile(state):
+    df = _state_df(state)
+    if df.empty:
+        return pd.DataFrame(), pd.DataFrame(), "No data loaded."
+    rows = []
+    for c in df.columns:
+        s = df[c]
+        flat = s.map(lambda v: B.stringify(v, 10_000))
+        rows.append({
+            "column": str(c),
+            "dtype": str(s.dtype),
+            "nulls": int(s.isna().sum()),
+            "null %": round(100 * s.isna().mean(), 2),
+            "unique": int(flat.nunique()),
+            "mean len": round(flat.str.len().mean(), 1),
+            "max len": int(flat.str.len().max()),
+            "example": B.stringify(s.dropna().iloc[0], 80) if s.notna().any() else "",
+        })
+    schema = pd.DataFrame(rows)
+
+    num = df.select_dtypes("number")
+    desc = num.describe().T.reset_index().rename(columns={"index": "column"}) if not num.empty \
+        else pd.DataFrame({"info": ["no numeric columns in this page"]})
+
+    # label-ish distribution
+    md = ["### Value distributions (low-cardinality columns)"]
+    for c in df.columns:
+        flat = df[c].map(lambda v: B.stringify(v, 60))
+        if 1 < flat.nunique() <= 25:
+            vc = flat.value_counts()
+            md.append(f"**{c}**")
+            md += [f"- `{k}` — {v} ({100*v/len(flat):.1f}%)" for k, v in vc.items()]
+    if len(md) == 1:
+        md.append("_none found on this page._")
+    return schema, desc, "\n".join(md)
+
+
+# --------------------------------------------------------------------------- #
+# tab 5: media gallery
+# --------------------------------------------------------------------------- #
+def gallery(state, col, n):
+    df = _state_df(state)
+    if df.empty:
+        return [], "No data loaded."
+    media = B.detect_media_columns(df)
+    col = col or (media["image"][0] if media["image"] else None)
+    if not col or col not in df.columns:
+        return [], "No image-like column detected on this page."
+    items = []
+    for i, v in df[col].head(int(n)).items():
+        d = B.media_to_display(v)
+        if d is not None:
+            items.append((d, f"#{i}"))
+    return items, f"{len(items)} image(s) from `{col}`"
+
+
+def media_choices(state):
+    df = _state_df(state)
+    if df.empty:
+        return gr.update(choices=[])
+    m = B.detect_media_columns(df)
+    return gr.update(choices=m["image"] + m["audio"],
+                     value=(m["image"] or m["audio"] or [None])[0])
+
+
+# --------------------------------------------------------------------------- #
+# tab 6: raw files
+# --------------------------------------------------------------------------- #
+def read_raw(dataset_meta, path, offset, limit, token, maxlen):
+    ds = (dataset_meta or {}).get("dataset", "")
+    if not ds or not path:
+        return pd.DataFrame(), "Pick a file first."
+    try:
+        res = B.read_raw_file(ds, path, token, int(limit), int(offset))
+    except Exception as e:  # noqa: BLE001
+        return pd.DataFrame(), f"❌ {e}"
+    return (B.display_frame(res.df, int(maxlen)),
+            f"**{res.strategy}** · {len(res.df)} rows"
+            + (f" of {res.total_rows:,}" if res.total_rows else ""))
+
+
+# --------------------------------------------------------------------------- #
+# tab 7: export
+# --------------------------------------------------------------------------- #
+def export(state, fmt):
+    df = _state_df(state)
+    if df.empty:
+        return None, "Nothing to export."
+    name = f"{(state or {}).get('dataset','hf')}".replace("/", "__") or "hf_export"
+    d = tempfile.mkdtemp()
+    path = os.path.join(d, f"{name}.{ {'CSV':'csv','JSON Lines':'jsonl','Parquet':'parquet'}[fmt] }")
+    try:
+        if fmt == "CSV":
+            B.display_frame(df, 10**6).to_csv(path, index=False)
+        elif fmt == "JSON Lines":
+            with open(path, "w", encoding="utf-8") as fh:
+                for _, r in df.iterrows():
+                    fh.write(json.dumps({str(k): B.json_safe(v) for k, v in r.items()},
+                                        ensure_ascii=False) + "\n")
+        else:
+            df.to_parquet(path, index=False)
+    except Exception as e:  # noqa: BLE001
+        return None, f"❌ {e}"
+    return path, f"✅ exported {len(df):,} rows"
+
+
+def code_snippet(meta, cfg, split, strategy):
+    ds = (meta or {}).get("dataset", "your/dataset")
+    return f'''```python
+# Reproduce this page in your own code
+from datasets import load_dataset
+
+# 1) streaming — works even when the Hub viewer is not ready
+ds = load_dataset("{ds}", {json.dumps(cfg) if cfg else "None"}, split="{split}", streaming=True)
+for i, ex in zip(range(5), ds):
+    print(ex)
+
+# 2) direct parquet (random access, no dataset script executed)
+import pandas as pd
+df = pd.read_parquet(
+    "hf://datasets/{ds}@refs/convert/parquet/{cfg or 'default'}/{split}/0000.parquet"
+)
+```'''
+
+
+# --------------------------------------------------------------------------- #
+# UI
+# --------------------------------------------------------------------------- #
+CSS = """
+.small-note {font-size: 0.85rem; opacity: .75}
+footer {display:none !important}
+"""
+
+with gr.Blocks(title="HF Dataset Explorer") as demo:
+    meta = gr.State({})
+    state = gr.State(dict(STATE_DEFAULT))
+
+    gr.Markdown(
+        "# 🤗 HF Dataset Explorer\n"
+        "View **any** Hugging Face dataset — even when the official dataset viewer "
+        "is not ready. Falls back automatically: `viewer API → parquet → streaming → raw files`."
+    )
+
+    with gr.Row():
+        dataset_id = gr.Textbox(label="Dataset id or URL", value="stanfordnlp/imdb",
+                                placeholder="user/name, or paste a huggingface.co URL", scale=4)
+        token = gr.Textbox(label="HF token (optional, for private/gated)",
+                           type="password", scale=2)
+        connect_btn = gr.Button("Connect", variant="primary", scale=1)
+    gr.Examples(EXAMPLES, inputs=dataset_id, label="Try")
+
+    with gr.Row():
+        config = gr.Dropdown(label="Config", choices=[], interactive=True, allow_custom_value=True)
+        split = gr.Dropdown(label="Split", choices=[], interactive=True, allow_custom_value=True)
+        strategy = gr.Dropdown(label="Loader", value="Auto (try everything)",
+                               choices=["Auto (try everything)", *B.STRATEGIES])
+        limit = gr.Slider(5, 500, 50, step=5, label="Rows per page")
+        offset = gr.Number(0, label="Offset", precision=0)
+        maxlen = gr.Slider(50, 2000, 300, step=50, label="Cell truncation (chars)")
+
+    info_md = gr.Markdown()
+
+    with gr.Tabs():
+        with gr.Tab("📋 Rows"):
+            with gr.Row():
+                load_btn = gr.Button("Load rows", variant="primary")
+                prev_btn = gr.Button("◀ Prev page")
+                next_btn = gr.Button("Next page ▶")
+            banner = gr.Markdown()
+            cols = gr.Dropdown(label="Visible columns", multiselect=True, choices=[], interactive=True)
+            table = gr.Dataframe(wrap=True, interactive=False, max_height=650)
+
+        with gr.Tab("🔎 Filter & search"):
+            gr.Markdown("Filters run on the **currently loaded page** (client-side, instant).")
+            with gr.Row():
+                query = gr.Textbox(label="pandas query", placeholder="label == 1 and len(text) > 500")
+                regex_col = gr.Dropdown(label="Regex column", choices=[], allow_custom_value=True)
+                regex = gr.Textbox(label="Regex / substring", placeholder="great|terrible")
+            filter_btn = gr.Button("Apply filter", variant="primary")
+            filter_note = gr.Markdown()
+            ftable = gr.Dataframe(wrap=True, interactive=False, max_height=600)
+
+        with gr.Tab("🔬 Record inspector"):
+            row_idx = gr.Slider(0, 1, 0, step=1, label="Row on this page")
+            with gr.Row():
+                with gr.Column(scale=3):
+                    rec_md = gr.Markdown()
+                with gr.Column(scale=2):
+                    rec_json = gr.JSON(label="Raw record")
+                    rec_img = gr.Image(label="Image field", height=260)
+                    rec_aud = gr.Audio(label="Audio field")
+
+        with gr.Tab("📊 Stats"):
+            stats_btn = gr.Button("Profile this page", variant="primary")
+            schema_tbl = gr.Dataframe(label="Schema & quality", wrap=True, interactive=False)
+            num_tbl = gr.Dataframe(label="Numeric describe()", interactive=False)
+            dist_md = gr.Markdown()
+
+        with gr.Tab("🖼️ Media"):
+            with gr.Row():
+                media_col = gr.Dropdown(label="Media column", choices=[], allow_custom_value=True)
+                n_media = gr.Slider(1, 100, 24, step=1, label="How many")
+                media_btn = gr.Button("Render", variant="primary")
+            media_note = gr.Markdown()
+            gal = gr.Gallery(columns=6, height=520, object_fit="contain")
+
+        with gr.Tab("🗂️ Repo files"):
+            gr.Markdown("Bypass everything and read a file straight out of the repo.")
+            files_tbl = gr.Dataframe(interactive=False, max_height=280)
+            with gr.Row():
+                file_pick = gr.Dropdown(label="Tabular file", choices=[], allow_custom_value=True, scale=3)
+                raw_off = gr.Number(0, label="Offset", precision=0)
+                raw_lim = gr.Slider(5, 500, 50, step=5, label="Rows")
+                raw_btn = gr.Button("Read file", variant="primary")
+            raw_note = gr.Markdown()
+            raw_tbl = gr.Dataframe(wrap=True, interactive=False, max_height=520)
+
+        with gr.Tab("📄 README"):
+            readme_md = gr.Markdown()
+
+        with gr.Tab("⬇️ Export & code"):
+            with gr.Row():
+                fmt = gr.Radio(["CSV", "JSON Lines", "Parquet"], value="CSV", label="Format")
+                exp_btn = gr.Button("Export current page", variant="primary")
+            exp_note = gr.Markdown()
+            exp_file = gr.File(label="Download")
+            snippet = gr.Markdown()
+
+    # ---------------- wiring ---------------- #
+    connect_btn.click(connect, [dataset_id, token],
+                      [config, split, info_md, readme_md, file_pick, meta]) \
+        .then(file_table, meta, files_tbl) \
+        .then(code_snippet, [meta, config, split, strategy], snippet)
+
+    dataset_id.submit(connect, [dataset_id, token],
+                      [config, split, info_md, readme_md, file_pick, meta]) \
+        .then(file_table, meta, files_tbl)
+
+    config.change(on_config_change, [config, meta], split)
+
+    load_inputs = [meta, config, split, offset, limit, token, strategy, maxlen, state]
+    load_btn.click(load_rows, load_inputs, [table, banner, state, cols, row_idx]) \
+        .then(lambda s: gr.update(choices=[str(c) for c in _state_df(s).columns]), state, regex_col) \
+        .then(media_choices, state, media_col) \
+        .then(code_snippet, [meta, config, split, strategy], snippet)
+
+    next_btn.click(lambda *a: page(1, *a), load_inputs,
+                   [table, banner, state, cols, row_idx, offset])
+    prev_btn.click(lambda *a: page(-1, *a), load_inputs,
+                   [table, banner, state, cols, row_idx, offset])
+
+    cols.change(pick_columns, [cols, state, maxlen], table)
+    filter_btn.click(apply_filter, [query, regex_col, regex, state, maxlen], [ftable, filter_note])
+    row_idx.change(inspect, [row_idx, state], [rec_json, rec_md, rec_img, rec_aud])
+    stats_btn.click(profile, state, [schema_tbl, num_tbl, dist_md])
+    media_btn.click(gallery, [state, media_col, n_media], [gal, media_note])
+    raw_btn.click(read_raw, [meta, file_pick, raw_off, raw_lim, token, maxlen], [raw_tbl, raw_note])
+    exp_btn.click(export, [state, fmt], [exp_file, exp_note])
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser()
+    p.add_argument("--share", action="store_true")
+    p.add_argument("--port", type=int, default=7860)
+    a = p.parse_args()
+    kwargs = dict(server_name="0.0.0.0", server_port=a.port,
+                  share=a.share, show_error=True)
+    try:  # gradio >= 6 moved theme/css to launch()
+        demo.queue(default_concurrency_limit=4).launch(
+            theme=gr.themes.Soft(), css=CSS, **kwargs)
+    except TypeError:
+        demo.queue(default_concurrency_limit=4).launch(**kwargs)