Upload README.md with huggingface_hub
Browse files
README.md
CHANGED
|
@@ -66,6 +66,37 @@ See the script at [demo_usage.py](demo_usage.py) for a quick start. You can run
|
|
| 66 |
python demo_usage.py
|
| 67 |
```
|
| 68 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
## Citation
|
| 70 |
|
| 71 |
If you use this model, please cite:
|
|
|
|
| 66 |
python demo_usage.py
|
| 67 |
```
|
| 68 |
|
| 69 |
+
OR use the snippet below:
|
| 70 |
+
|
| 71 |
+
```python
|
| 72 |
+
import torch
|
| 73 |
+
from modeling_tara import TARA, read_frames_decord
|
| 74 |
+
|
| 75 |
+
model = TARA.from_pretrained(
|
| 76 |
+
".", # Load from current directory
|
| 77 |
+
device_map='auto',
|
| 78 |
+
torch_dtype=torch.bfloat16,
|
| 79 |
+
)
|
| 80 |
+
n_params = sum(p.numel() for p in model.model.parameters())
|
| 81 |
+
print(f"Number of parameters: {round(n_params/1e9, 3)}B")
|
| 82 |
+
|
| 83 |
+
# Embed a video
|
| 84 |
+
video_path = "./assets/folding_paper.mp4"
|
| 85 |
+
video_tensor = read_frames_decord(video_path, num_frames=16)
|
| 86 |
+
video_tensor = video_tensor.unsqueeze(0)
|
| 87 |
+
video_tensor = video_tensor.to(model.model.device)
|
| 88 |
+
with torch.no_grad():
|
| 89 |
+
video_emb = model.encode_vision(video_tensor).cpu().squeeze(0).float()
|
| 90 |
+
print(f"Video shape: {video_tensor.shape}") # torch.Size([1, 16, 3, 240, 426])
|
| 91 |
+
print(f"Video embedding shape: {video_emb.shape}") # torch.Size([4096])
|
| 92 |
+
|
| 93 |
+
# Embed a text
|
| 94 |
+
text = ['someone is folding a paper', 'cutting a paper', 'someone is folding a paper']
|
| 95 |
+
with torch.no_grad():
|
| 96 |
+
text_emb = model.encode_text(text).cpu().float()
|
| 97 |
+
print(f"Text embedding shape: {text_emb.shape}") # torch.Size([3, 4096])
|
| 98 |
+
```
|
| 99 |
+
|
| 100 |
## Citation
|
| 101 |
|
| 102 |
If you use this model, please cite:
|