multimodalart HF Staff commited on
Commit
a169d4a
·
1 Parent(s): b9e7e6d

[Admin maintenance] Support new ZeroGPU hardware (#3)

Browse files

- [Admin maintenance] Support new ZeroGPU hardware (c1353441352a60bd9980dd5ad32cb9340edd30ac)

Files changed (3) hide show
  1. README.md +1 -1
  2. app.py +111 -8
  3. requirements.txt +5 -8
README.md CHANGED
@@ -4,7 +4,7 @@ emoji: 💻
4
  colorFrom: yellow
5
  colorTo: indigo
6
  sdk: gradio
7
- sdk_version: 4.31.0
8
  app_file: app.py
9
  license: mit
10
  pinned: false
 
4
  colorFrom: yellow
5
  colorTo: indigo
6
  sdk: gradio
7
+ sdk_version: 5.49.1
8
  app_file: app.py
9
  license: mit
10
  pinned: false
app.py CHANGED
@@ -1,11 +1,116 @@
1
  import os
2
- import subprocess
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
 
4
- subprocess.run(
5
- "pip install flash-attn==2.7.4.post1 --no-build-isolation",
6
- env={"FLASH_ATTENTION_SKIP_CUDA_BUILD": "TRUE"},
7
- shell=True,
8
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9
 
10
  os.makedirs("/home/user/app/checkpoints", exist_ok=True)
11
  from huggingface_hub import snapshot_download
@@ -20,13 +125,11 @@ import builtins
20
  import json
21
  import math
22
  import multiprocessing as mp
23
- import os
24
  import random
25
  import socket
26
  import traceback
27
 
28
  from PIL import Image
29
- import spaces
30
  import gradio as gr
31
  import numpy as np
32
  from safetensors.torch import load_file
 
1
  import os
2
+ import sys
3
+ import types
4
+
5
+ # Spaces must be imported before any CUDA-touching package.
6
+ import spaces
7
+
8
+ # Install a flash_attn SDPA-fallback shim BEFORE `import models` (which does
9
+ # `from flash_attn import flash_attn_varlen_func`). The prebuilt flash-attn
10
+ # wheels do not run on Blackwell sm_120 / CUDA 13, so we replace the only two
11
+ # symbols the model imports (`flash_attn_varlen_func` and the bert_padding
12
+ # helpers) with torch-native equivalents.
13
+ import math as _math
14
+ import torch as _torch
15
+ import torch.nn.functional as _F
16
+
17
+
18
+ def _flash_attn_varlen_func(
19
+ q, k, v, cu_seqlens_q, cu_seqlens_k, max_seqlen_q, max_seqlen_k,
20
+ dropout_p=0.0, softmax_scale=None, causal=False, **_kwargs,
21
+ ):
22
+ # q/k/v are [total_tokens, n_heads, head_dim], packed per batch row.
23
+ # Lumina-Next uses GQA: q has n_heads, k/v have n_kv_heads < n_heads.
24
+ # Repeat k/v along the head dim so SDPA sees matching shapes.
25
+ total_q, n_heads, head_dim = q.shape
26
+ n_kv_heads = k.shape[1]
27
+ out = _torch.empty_like(q)
28
+ cu_q = cu_seqlens_q.tolist()
29
+ cu_k = cu_seqlens_k.tolist()
30
+ repeat = n_heads // n_kv_heads if n_kv_heads > 0 else 1
31
+ for b in range(len(cu_q) - 1):
32
+ qs, qe = cu_q[b], cu_q[b + 1]
33
+ ks, ke = cu_k[b], cu_k[b + 1]
34
+ if qe == qs:
35
+ continue
36
+ qb = q[qs:qe].transpose(0, 1).unsqueeze(0) # [1, n_heads, lq, d]
37
+ kb = k[ks:ke].transpose(0, 1).unsqueeze(0) # [1, n_kv, lk, d]
38
+ vb = v[ks:ke].transpose(0, 1).unsqueeze(0)
39
+ if repeat > 1:
40
+ kb = kb.repeat_interleave(repeat, dim=1)
41
+ vb = vb.repeat_interleave(repeat, dim=1)
42
+ ob = _F.scaled_dot_product_attention(
43
+ qb, kb, vb,
44
+ dropout_p=dropout_p,
45
+ scale=softmax_scale,
46
+ is_causal=causal,
47
+ )
48
+ out[qs:qe] = ob.squeeze(0).transpose(0, 1)
49
+ return out
50
+
51
+
52
+ def _index_first_axis(input_, indices):
53
+ return input_[indices]
54
 
55
+
56
+ def _unpad_input(hidden_states, attention_mask):
57
+ # hidden_states: [bsz, seqlen, ...]; attention_mask: [bsz, seqlen] (bool/int)
58
+ bsz, seqlen = attention_mask.shape[:2]
59
+ seqlens_in_batch = attention_mask.sum(dim=-1, dtype=_torch.int32)
60
+ indices = _torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
61
+ max_seqlen_in_batch = int(seqlens_in_batch.max().item()) if bsz > 0 else 0
62
+ cu_seqlens = _F.pad(_torch.cumsum(seqlens_in_batch, dim=0, dtype=_torch.int32), (1, 0))
63
+ flat = hidden_states.reshape(bsz * seqlen, *hidden_states.shape[2:])
64
+ return flat[indices], indices, cu_seqlens, max_seqlen_in_batch
65
+
66
+
67
+ def _pad_input(hidden_states, indices, batch, seqlen):
68
+ output = _torch.zeros(
69
+ batch * seqlen, *hidden_states.shape[1:],
70
+ device=hidden_states.device, dtype=hidden_states.dtype,
71
+ )
72
+ output[indices] = hidden_states
73
+ return output.reshape(batch, seqlen, *hidden_states.shape[1:])
74
+
75
+
76
+ import importlib.machinery as _machinery
77
+ _flash_attn_mod = types.ModuleType("flash_attn")
78
+ _flash_attn_mod.__spec__ = _machinery.ModuleSpec("flash_attn", loader=None)
79
+ _flash_attn_mod.__version__ = "0.0.0-shim"
80
+ _flash_attn_mod.flash_attn_varlen_func = _flash_attn_varlen_func
81
+ _bert_padding_mod = types.ModuleType("flash_attn.bert_padding")
82
+ _bert_padding_mod.__spec__ = _machinery.ModuleSpec("flash_attn.bert_padding", loader=None)
83
+ _bert_padding_mod.index_first_axis = _index_first_axis
84
+ _bert_padding_mod.pad_input = _pad_input
85
+ _bert_padding_mod.unpad_input = _unpad_input
86
+ _flash_attn_mod.bert_padding = _bert_padding_mod
87
+ sys.modules["flash_attn"] = _flash_attn_mod
88
+ sys.modules["flash_attn.bert_padding"] = _bert_padding_mod
89
+
90
+ # torch.load weights_only=False shim — old checkpoints (model_args.pth) contain
91
+ # argparse.Namespace pickles that fail under torch>=2.6 default weights_only=True.
92
+ _orig_torch_load = _torch.load
93
+ _torch.load = lambda *a, **k: _orig_torch_load(*a, **{**k, "weights_only": k.get("weights_only", False)})
94
+
95
+ # Modern transformers' is_flash_attn_*_available() consults
96
+ # PACKAGE_DISTRIBUTION_MAPPING["flash_attn"] when `flash_attn` is importable;
97
+ # our shim has no real distribution metadata, so the lookup throws KeyError
98
+ # and cascades through diffusers' lazy import machinery.
99
+ # Register `flash_attn` in that mapping AND force every flash-attn detection
100
+ # helper to report False so transformers/diffusers go down the SDPA path.
101
+ import transformers # noqa: E402
102
+ from transformers.utils import import_utils as _tu # noqa: E402
103
+ if hasattr(_tu, "PACKAGE_DISTRIBUTION_MAPPING"):
104
+ _tu.PACKAGE_DISTRIBUTION_MAPPING.setdefault("flash_attn", ["flash-attn"])
105
+ for _name in (
106
+ "is_flash_attn_2_available",
107
+ "is_flash_attn_3_available",
108
+ "is_flash_attn_4_available",
109
+ "is_flash_attn_greater_or_equal_2_10",
110
+ "is_flash_attn_greater_or_equal",
111
+ ):
112
+ if hasattr(_tu, _name):
113
+ setattr(_tu, _name, lambda *a, **k: False)
114
 
115
  os.makedirs("/home/user/app/checkpoints", exist_ok=True)
116
  from huggingface_hub import snapshot_download
 
125
  import json
126
  import math
127
  import multiprocessing as mp
 
128
  import random
129
  import socket
130
  import traceback
131
 
132
  from PIL import Image
 
133
  import gradio as gr
134
  import numpy as np
135
  from safetensors.torch import load_file
requirements.txt CHANGED
@@ -1,15 +1,12 @@
1
- --index-url https://download.pytorch.org/whl/cu118
2
- torch==2.2.0
3
- torchvision==0.17.0
4
- torchaudio==2.2.0
5
- --index-url https://pypi.org/simple
6
- transformers==4.40.2
7
- diffusers==0.26.3
8
  spaces
9
  fairscale
10
  numpy
11
  pillow
12
  torchdiffeq
13
  click
 
 
 
14
  git+https://github.com/Alpha-VLLM/Lumina-T2X
15
- huggingface_hub==0.25.0
 
1
+ transformers
2
+ diffusers>=0.30
 
 
 
 
 
3
  spaces
4
  fairscale
5
  numpy
6
  pillow
7
  torchdiffeq
8
  click
9
+ torchvision
10
+ safetensors
11
+ sentencepiece
12
  git+https://github.com/Alpha-VLLM/Lumina-T2X