Spaces:
Running on Zero
Running on Zero
Commit ·
a169d4a
1
Parent(s): b9e7e6d
[Admin maintenance] Support new ZeroGPU hardware (#3)
Browse files- [Admin maintenance] Support new ZeroGPU hardware (c1353441352a60bd9980dd5ad32cb9340edd30ac)
- README.md +1 -1
- app.py +111 -8
- requirements.txt +5 -8
README.md
CHANGED
|
@@ -4,7 +4,7 @@ emoji: 💻
|
|
| 4 |
colorFrom: yellow
|
| 5 |
colorTo: indigo
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version:
|
| 8 |
app_file: app.py
|
| 9 |
license: mit
|
| 10 |
pinned: false
|
|
|
|
| 4 |
colorFrom: yellow
|
| 5 |
colorTo: indigo
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: 5.49.1
|
| 8 |
app_file: app.py
|
| 9 |
license: mit
|
| 10 |
pinned: false
|
app.py
CHANGED
|
@@ -1,11 +1,116 @@
|
|
| 1 |
import os
|
| 2 |
-
import
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
|
| 10 |
os.makedirs("/home/user/app/checkpoints", exist_ok=True)
|
| 11 |
from huggingface_hub import snapshot_download
|
|
@@ -20,13 +125,11 @@ import builtins
|
|
| 20 |
import json
|
| 21 |
import math
|
| 22 |
import multiprocessing as mp
|
| 23 |
-
import os
|
| 24 |
import random
|
| 25 |
import socket
|
| 26 |
import traceback
|
| 27 |
|
| 28 |
from PIL import Image
|
| 29 |
-
import spaces
|
| 30 |
import gradio as gr
|
| 31 |
import numpy as np
|
| 32 |
from safetensors.torch import load_file
|
|
|
|
| 1 |
import os
|
| 2 |
+
import sys
|
| 3 |
+
import types
|
| 4 |
+
|
| 5 |
+
# Spaces must be imported before any CUDA-touching package.
|
| 6 |
+
import spaces
|
| 7 |
+
|
| 8 |
+
# Install a flash_attn SDPA-fallback shim BEFORE `import models` (which does
|
| 9 |
+
# `from flash_attn import flash_attn_varlen_func`). The prebuilt flash-attn
|
| 10 |
+
# wheels do not run on Blackwell sm_120 / CUDA 13, so we replace the only two
|
| 11 |
+
# symbols the model imports (`flash_attn_varlen_func` and the bert_padding
|
| 12 |
+
# helpers) with torch-native equivalents.
|
| 13 |
+
import math as _math
|
| 14 |
+
import torch as _torch
|
| 15 |
+
import torch.nn.functional as _F
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _flash_attn_varlen_func(
|
| 19 |
+
q, k, v, cu_seqlens_q, cu_seqlens_k, max_seqlen_q, max_seqlen_k,
|
| 20 |
+
dropout_p=0.0, softmax_scale=None, causal=False, **_kwargs,
|
| 21 |
+
):
|
| 22 |
+
# q/k/v are [total_tokens, n_heads, head_dim], packed per batch row.
|
| 23 |
+
# Lumina-Next uses GQA: q has n_heads, k/v have n_kv_heads < n_heads.
|
| 24 |
+
# Repeat k/v along the head dim so SDPA sees matching shapes.
|
| 25 |
+
total_q, n_heads, head_dim = q.shape
|
| 26 |
+
n_kv_heads = k.shape[1]
|
| 27 |
+
out = _torch.empty_like(q)
|
| 28 |
+
cu_q = cu_seqlens_q.tolist()
|
| 29 |
+
cu_k = cu_seqlens_k.tolist()
|
| 30 |
+
repeat = n_heads // n_kv_heads if n_kv_heads > 0 else 1
|
| 31 |
+
for b in range(len(cu_q) - 1):
|
| 32 |
+
qs, qe = cu_q[b], cu_q[b + 1]
|
| 33 |
+
ks, ke = cu_k[b], cu_k[b + 1]
|
| 34 |
+
if qe == qs:
|
| 35 |
+
continue
|
| 36 |
+
qb = q[qs:qe].transpose(0, 1).unsqueeze(0) # [1, n_heads, lq, d]
|
| 37 |
+
kb = k[ks:ke].transpose(0, 1).unsqueeze(0) # [1, n_kv, lk, d]
|
| 38 |
+
vb = v[ks:ke].transpose(0, 1).unsqueeze(0)
|
| 39 |
+
if repeat > 1:
|
| 40 |
+
kb = kb.repeat_interleave(repeat, dim=1)
|
| 41 |
+
vb = vb.repeat_interleave(repeat, dim=1)
|
| 42 |
+
ob = _F.scaled_dot_product_attention(
|
| 43 |
+
qb, kb, vb,
|
| 44 |
+
dropout_p=dropout_p,
|
| 45 |
+
scale=softmax_scale,
|
| 46 |
+
is_causal=causal,
|
| 47 |
+
)
|
| 48 |
+
out[qs:qe] = ob.squeeze(0).transpose(0, 1)
|
| 49 |
+
return out
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _index_first_axis(input_, indices):
|
| 53 |
+
return input_[indices]
|
| 54 |
|
| 55 |
+
|
| 56 |
+
def _unpad_input(hidden_states, attention_mask):
|
| 57 |
+
# hidden_states: [bsz, seqlen, ...]; attention_mask: [bsz, seqlen] (bool/int)
|
| 58 |
+
bsz, seqlen = attention_mask.shape[:2]
|
| 59 |
+
seqlens_in_batch = attention_mask.sum(dim=-1, dtype=_torch.int32)
|
| 60 |
+
indices = _torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
|
| 61 |
+
max_seqlen_in_batch = int(seqlens_in_batch.max().item()) if bsz > 0 else 0
|
| 62 |
+
cu_seqlens = _F.pad(_torch.cumsum(seqlens_in_batch, dim=0, dtype=_torch.int32), (1, 0))
|
| 63 |
+
flat = hidden_states.reshape(bsz * seqlen, *hidden_states.shape[2:])
|
| 64 |
+
return flat[indices], indices, cu_seqlens, max_seqlen_in_batch
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _pad_input(hidden_states, indices, batch, seqlen):
|
| 68 |
+
output = _torch.zeros(
|
| 69 |
+
batch * seqlen, *hidden_states.shape[1:],
|
| 70 |
+
device=hidden_states.device, dtype=hidden_states.dtype,
|
| 71 |
+
)
|
| 72 |
+
output[indices] = hidden_states
|
| 73 |
+
return output.reshape(batch, seqlen, *hidden_states.shape[1:])
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
import importlib.machinery as _machinery
|
| 77 |
+
_flash_attn_mod = types.ModuleType("flash_attn")
|
| 78 |
+
_flash_attn_mod.__spec__ = _machinery.ModuleSpec("flash_attn", loader=None)
|
| 79 |
+
_flash_attn_mod.__version__ = "0.0.0-shim"
|
| 80 |
+
_flash_attn_mod.flash_attn_varlen_func = _flash_attn_varlen_func
|
| 81 |
+
_bert_padding_mod = types.ModuleType("flash_attn.bert_padding")
|
| 82 |
+
_bert_padding_mod.__spec__ = _machinery.ModuleSpec("flash_attn.bert_padding", loader=None)
|
| 83 |
+
_bert_padding_mod.index_first_axis = _index_first_axis
|
| 84 |
+
_bert_padding_mod.pad_input = _pad_input
|
| 85 |
+
_bert_padding_mod.unpad_input = _unpad_input
|
| 86 |
+
_flash_attn_mod.bert_padding = _bert_padding_mod
|
| 87 |
+
sys.modules["flash_attn"] = _flash_attn_mod
|
| 88 |
+
sys.modules["flash_attn.bert_padding"] = _bert_padding_mod
|
| 89 |
+
|
| 90 |
+
# torch.load weights_only=False shim — old checkpoints (model_args.pth) contain
|
| 91 |
+
# argparse.Namespace pickles that fail under torch>=2.6 default weights_only=True.
|
| 92 |
+
_orig_torch_load = _torch.load
|
| 93 |
+
_torch.load = lambda *a, **k: _orig_torch_load(*a, **{**k, "weights_only": k.get("weights_only", False)})
|
| 94 |
+
|
| 95 |
+
# Modern transformers' is_flash_attn_*_available() consults
|
| 96 |
+
# PACKAGE_DISTRIBUTION_MAPPING["flash_attn"] when `flash_attn` is importable;
|
| 97 |
+
# our shim has no real distribution metadata, so the lookup throws KeyError
|
| 98 |
+
# and cascades through diffusers' lazy import machinery.
|
| 99 |
+
# Register `flash_attn` in that mapping AND force every flash-attn detection
|
| 100 |
+
# helper to report False so transformers/diffusers go down the SDPA path.
|
| 101 |
+
import transformers # noqa: E402
|
| 102 |
+
from transformers.utils import import_utils as _tu # noqa: E402
|
| 103 |
+
if hasattr(_tu, "PACKAGE_DISTRIBUTION_MAPPING"):
|
| 104 |
+
_tu.PACKAGE_DISTRIBUTION_MAPPING.setdefault("flash_attn", ["flash-attn"])
|
| 105 |
+
for _name in (
|
| 106 |
+
"is_flash_attn_2_available",
|
| 107 |
+
"is_flash_attn_3_available",
|
| 108 |
+
"is_flash_attn_4_available",
|
| 109 |
+
"is_flash_attn_greater_or_equal_2_10",
|
| 110 |
+
"is_flash_attn_greater_or_equal",
|
| 111 |
+
):
|
| 112 |
+
if hasattr(_tu, _name):
|
| 113 |
+
setattr(_tu, _name, lambda *a, **k: False)
|
| 114 |
|
| 115 |
os.makedirs("/home/user/app/checkpoints", exist_ok=True)
|
| 116 |
from huggingface_hub import snapshot_download
|
|
|
|
| 125 |
import json
|
| 126 |
import math
|
| 127 |
import multiprocessing as mp
|
|
|
|
| 128 |
import random
|
| 129 |
import socket
|
| 130 |
import traceback
|
| 131 |
|
| 132 |
from PIL import Image
|
|
|
|
| 133 |
import gradio as gr
|
| 134 |
import numpy as np
|
| 135 |
from safetensors.torch import load_file
|
requirements.txt
CHANGED
|
@@ -1,15 +1,12 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
torchvision==0.17.0
|
| 4 |
-
torchaudio==2.2.0
|
| 5 |
-
--index-url https://pypi.org/simple
|
| 6 |
-
transformers==4.40.2
|
| 7 |
-
diffusers==0.26.3
|
| 8 |
spaces
|
| 9 |
fairscale
|
| 10 |
numpy
|
| 11 |
pillow
|
| 12 |
torchdiffeq
|
| 13 |
click
|
|
|
|
|
|
|
|
|
|
| 14 |
git+https://github.com/Alpha-VLLM/Lumina-T2X
|
| 15 |
-
huggingface_hub==0.25.0
|
|
|
|
| 1 |
+
transformers
|
| 2 |
+
diffusers>=0.30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
spaces
|
| 4 |
fairscale
|
| 5 |
numpy
|
| 6 |
pillow
|
| 7 |
torchdiffeq
|
| 8 |
click
|
| 9 |
+
torchvision
|
| 10 |
+
safetensors
|
| 11 |
+
sentencepiece
|
| 12 |
git+https://github.com/Alpha-VLLM/Lumina-T2X
|
|
|