Initial commit

to upload the model

README.md

@@ -1,3 +1,44 @@
 ---
 license: mit
+library_name: transformers
+tags:
+- Robotics
+- Vision-Language-Action
+- Manipulation
+- Multimodal
+- Pretraining
+- Diffusion
+language:
+- en
+pipeline_tag: Robotics
 ---
+# VITRA-VLA-3B
+VITRA is a novel approach for pretraining Vision-Language-Action (VLA) models for robotic manipulation using large-scale, unscripted, real-world videos of human hand activities. Treating human hand as dexterous robot end-effector, we show that in-the-wild egocentric human videos without any annotations can be transformed into data formats fully aligned with existing robotic V-L-A training data in terms of task granularity and labels. We create a human hand V-L-A dataset containing over 1 million episodes. We further develop a VLA model with a causal action transformer trained on this dataset. It demonstrates strong zero-shot human-hand action prediction in entirely new scenes and serves as a cornerstone for few-shot finetuning and adaptation to real-world robotic manipulation.
+
+
+All our [code](https://github.com/microsoft/VITRA), [pre-trained model weights](https://huggingface.co/microsoft/VITRA-VLA-3B), are licensed under the MIT license.
+
+Please refer to our [project page](https://microsoft.github.io/VITRA/) and [paper](https://arxiv.org/abs/2510.21571) for more details.
+
+
+## Model Summary
+
+- **Developed by:** The VITRA team consisting of researchers from [Microsoft Research](https://www.microsoft.com/en-us/research/lab/microsoft-research-asia/).
+- **Model type:** Vision-Language-Action (language, image => robot actions)
+- **Language(s) (NLP):** en
+- **License:** MIT
+- **Pretraining Dataset:** [VITRA-1M](https://huggingface.co/datasets/VITRA-VLA/VITRA-1M)
+- **Repository:** [https://github.com/microsoft/ViTra](https://github.com/microsoft/VITRA)
+- **Paper:** [Scalable Vision-Language-Action Model Pretraining for Robotic Manipulation with Real-Life Human Activity Videos](https://arxiv.org/abs/2510.21571)
+- **Project Page:** [https://microsoft.github.io/VITRA/](https://microsoft.github.io/VITRA/)
+
+## Citation
+
+```bibtex
+@article{li2025vitra,
+  title={Scalable Vision-Language-Action Model Pretraining for Robotic Manipulation with Real-Life Human Activity Videos},
+  author={Qixiu Li and Yu Deng and Yaobo Liang and Lin Luo and Lei Zhou and Chengtang Yao and Lingqi Zeng and Zhiyuan Feng and Huizhi Liang and Sicheng Xu and Yizhong Zhang and Xi Chen and Hao Chen and Lily Sun and Dong Chen and Jiaolong Yang and Baining Guo},
+  journal={arXiv preprint arXiv:2510.21571},
+  year={2025}
+}
+```

config.json

@@ -0,0 +1,75 @@
+{
+  "vla_name": "VITRA_Paligemma",
+  "task_name": "pretrain",
+  "model": "vitra_paligemma2",
+  "fwd_pred_next_n": 16,
+  "seed": 42,
+  "batch_size": 64,
+  "output_root": "/data/vla_checkpoint/vitra_vla_3b/checkpoints",
+  "log_root": "/data/vla_checkpoint/vitra_vla_3b/logs",
+  "cache_root": "/data/vla_checkpoint/vitra_vla_3b/cache/vitra_paligemma2",
+  "model_load_path": null,
+  "resume": true,
+  "wandb_project": "vitra_paligemma2_humanpretrain",
+  "wandb_entity": "",
+  "save_steps": 5000,
+  "total_batch_size": 512,
+  "use_bf16": true,
+  "use_fov": true,
+  "untied_cognition_token": true,
+  "use_state": "DiT",
+  "loss_type": "human",
+  "train_setup": {
+    "freeze_option": "freeze_vision_encoder"
+  },
+  "state_encoder": {
+    "state_dim": 212
+  },
+  "action_model": {
+    "model_type": "DiT-B",
+    "token_size": 2304,
+    "action_dim": 192,
+    "hidden_size": 1024
+  },
+  "vlm": {
+    "type": "PaliGemmaForConditionalGeneration",
+    "name": "paligemma",
+    "pretrained_model_name_or_path": "google/paligemma2-3b-mix-224"
+  },
+  "trainer": {
+    "sharding_strategy": "shard-grad-op",
+    "strategy": "fsdp_paligemma_with_checkpointing",
+    "lr_scheduler_type": "backbone-freeze-warmup",
+    "gradient_clip_val": 1.0,
+    "learning_rate": 1e-05,
+    "weight_decay": 0.1,
+    "max_epochs": 100000,
+    "max_steps": 2000000,
+    "reduce_in_full_precision": true,
+    "enable_mixed_precision_training": false,
+    "enable_gradient_checkpointing": true,
+    "action_model_learning_rate": 0.0001,
+    "llm_freeze_step": 5000,
+    "warmup_ratio": null
+  },
+  "train_dataset": {
+    "data_root_dir": "/data/VITRA_1M",
+    "augmentation": true,
+    "set_none_ratio": 0.0,
+    "data_mix": "magic_mix",
+    "num_workers": 18,
+    "prefetch_factor": null,
+    "flip_augmentation": 1.0,
+    "action_type": "angle",
+    "use_rel": false,
+    "clip_len": 2000,
+    "normalization": true,
+    "state_mask_prob": 0.1
+  },
+  "repeated_diffusion_steps": 8,
+  "config": "vitra/configs/human_pretrain.json",
+  "data_mix": null,
+  "debug": false,
+  "num_workers": null,
+  "prefetch_factor": null
+}

dataset_statistics.json

@@ -0,0 +1,475 @@
+{
+  "dataset_name": "magic_mix_angle",
+  "state_left": {
+    "mean": [
+      -0.11579255147603508,
+      0.09853558399019087,
+      0.44707650050681974,
+      -1.2541853618288576,
+      -0.4000807535824712,
+      -1.2473501749009082,
+      0.0014194980813503951,
+      -0.20215592029794455,
+      0.4881023094619483,
+      0.144666114219659,
+      -0.020534433704960837,
+      0.7648325610127158,
+      -0.061935409523992874,
+      0.12219282915289578,
+      0.26113551921252837,
+      -0.18055030545219955,
+      0.00818508913055498,
+      0.8038736724790866,
+      -0.1813954125900856,
+      -0.05255599370465047,
+      0.615207956050913,
+      0.019420943883633742,
+      0.08206648678538951,
+      0.48009611970270755,
+      -0.29267464483981337,
+      0.5924368191514214,
+      0.6207362092964281,
+      -0.25156062915694505,
+      0.042651493029024115,
+      0.48423217349715236,
+      -0.334746772726311,
+      0.15674395989629697,
+      0.29122580010435584,
+      0.002590410652552483,
+      0.1964149148587996,
+      0.835560202060088,
+      -0.36014970866233637,
+      0.18172483650834523,
+      0.46660463448032335,
+      -0.1580085402873619,
+      0.1527547535010859,
+      0.56291991927379,
+      0.940616712603359,
+      -0.257409582255471,
+      -0.17166173691980588,
+      -0.5820884370125198,
+      -0.05102693864175229,
+      0.08250452865880151,
+      0.5624305682602475,
+      -0.10296209321668819,
+      0.1503018356014215,
+      -0.5747307150940298,
+      0.23351668183581697,
+      -1.1903627395932146,
+      -0.9084577659186075,
+      -0.6462851164081945,
+      -1.3938379994860928,
+      -1.0614910076816242,
+      0.30291840524363217,
+      0.36528067170784034,
+      1.1302787101320517
+    ],
+    "std": [
+      0.11614162849409655,
+      0.11901126557680539,
+      0.12419630282197135,
+      2.0007552999619778,
+      0.45448972901077384,
+      0.7817598955940285,
+      0.11128080284509624,
+      0.12912182093439675,
+      0.35545058646100175,
+      0.12876485704777554,
+      0.08435337718455918,
+      0.36610667194321145,
+      0.13724484904033574,
+      0.10688082517603581,
+      0.26567648172114117,
+      0.16147559793613112,
+      0.12473006685795233,
+      0.44581730434535316,
+      0.07306171120288285,
+      0.08736870809283682,
+      0.31900790613664437,
+      0.072888562546578,
+      0.06399497127762747,
+      0.2663932808678995,
+      0.33543888868189736,
+      0.21170762055192094,
+      0.3619610208163317,
+      0.35235464267160205,
+      0.2084891628496783,
+      0.2235456188527172,
+      0.22744935255345794,
+      0.1885276766999644,
+      0.3418175798449056,
+      0.12194798172893641,
+      0.1256308361024914,
+      0.4346914289200111,
+      0.08953834964768714,
+      0.15567065816119383,
+      0.3180041158189954,
+      0.09816140209765825,
+      0.08802720644673442,
+      0.2866974922019247,
+      0.2929987207576424,
+      0.20324254048887913,
+      0.15323708166010291,
+      0.17404420224680545,
+      0.1945877494178962,
+      0.3252053447856269,
+      0.22420596660717013,
+      0.28561784696450015,
+      0.2853439385961586,
+      0.6396521245963158,
+      0.2582481965386295,
+      0.6770440958787715,
+      0.32181806723846684,
+      0.17694618334172266,
+      0.5708438664188205,
+      0.23984804504988888,
+      0.16358736096578716,
+      0.17581568964731845,
+      0.15743956600317066
+    ]
+  },
+  "action_left": {
+    "mean": [
+      9.201280270413256e-05,
+      -0.0004563322208652859,
+      0.0003617891466070917,
+      -0.0003427012188614409,
+      -0.0002564684738122795,
+      0.00011712473943147219,
+      0.0014139227671159532,
+      -0.20215362312171495,
+      0.4883238904456818,
+      0.14473984845079738,
+      -0.020567718621431218,
+      0.7651154595792194,
+      -0.061976103515996014,
+      0.12226727847795801,
+      0.26117868914664694,
+      -0.18061434692865094,
+      0.00826280607268428,
+      0.8041355462487079,
+      -0.18137487653545137,
+      -0.05257555295826352,
+      0.6153581688280276,
+      0.01942984430339739,
+      0.08211296174733909,
+      0.4802630569776921,
+      -0.2927880225943608,
+      0.5925688028379226,
+      0.6207528795340029,
+      -0.2515588953494671,
+      0.042636435625136485,
+      0.48434184787188456,
+      -0.3348895027302863,
+      0.15671903162520343,
+      0.2913429105139313,
+      0.0026171841327992705,
+      0.19648877443263496,
+      0.8357650843097173,
+      -0.3601878069085853,
+      0.1818397955054986,
+      0.46672227904389374,
+      -0.15806518022867352,
+      0.15275533037946065,
+      0.5630445929293807,
+      0.9408879190240531,
+      -0.25754508183600444,
+      -0.17176383718742938,
+      -0.5822175956770262,
+      -0.05109248819502697,
+      0.08240467902412843,
+      0.5625306126074148,
+      -0.10280732140233888,
+      0.15035902087164887
+    ],
+    "std": [
+      0.007177335636993517,
+      0.007323014615215487,
+      0.007108638558558272,
+      0.03838344220758514,
+      0.040972733963894115,
+      0.04106513995602905,
+      0.11129771071505476,
+      0.12913963512346055,
+      0.355464778471544,
+      0.1287554216849025,
+      0.08436753183315916,
+      0.3661838001000936,
+      0.1372491056452125,
+      0.10690078174246394,
+      0.2657842622899636,
+      0.1614754862158314,
+      0.1247228132417261,
+      0.44582370853114633,
+      0.07306784405186532,
+      0.08739010735285878,
+      0.3191330269342099,
+      0.07288735784692742,
+      0.06400415596108433,
+      0.26651046131879086,
+      0.3355727061159244,
+      0.21172997529909793,
+      0.36200594576304107,
+      0.3525150130471941,
+      0.20857451173150765,
+      0.22362183701041413,
+      0.22756636946981657,
+      0.18861742937771767,
+      0.34191283757471225,
+      0.12199086416574195,
+      0.12559905526299284,
+      0.43476334926719845,
+      0.08954847401088233,
+      0.1557126939051543,
+      0.3181014101167722,
+      0.09817283234797235,
+      0.08804499244769111,
+      0.2867628887364027,
+      0.29309290272774746,
+      0.20317565434546034,
+      0.15325299974903256,
+      0.17404971850390882,
+      0.19461596175157847,
+      0.325123111971396,
+      0.22423240037688688,
+      0.28573026062918094,
+      0.28530750875194993
+    ]
+  },
+  "state_right": {
+    "mean": [
+      0.11585512642579204,
+      0.09852785475620146,
+      0.4472017838882995,
+      -1.2546634294247712,
+      0.39996407665917405,
+      1.24778099549795,
+      0.001412634035388727,
+      -0.2021876628633721,
+      0.48792879834001157,
+      0.14459316525542248,
+      -0.02052885803866319,
+      0.7649226888195222,
+      -0.061873089249993636,
+      0.12219067022454297,
+      0.26126941481972893,
+      -0.18057174367622797,
+      0.008144257601891492,
+      0.8037431575359127,
+      -0.1813684622351691,
+      -0.05254197165020591,
+      0.6153151241999176,
+      0.019415560460376794,
+      0.08206879005769253,
+      0.4800865894191521,
+      -0.29262389416926055,
+      0.5922639840783065,
+      0.6205970068957954,
+      -0.25158606978364245,
+      0.042637170174102665,
+      0.48423335539506124,
+      -0.3348539197622029,
+      0.15674491435097035,
+      0.29117609757999247,
+      0.0025620085401069992,
+      0.19633490606450155,
+      0.8353808980400095,
+      -0.36011263555662215,
+      0.1817254239823724,
+      0.4667154358950734,
+      -0.15802000480938058,
+      0.15278471908271476,
+      0.562938927891565,
+      0.9406496558464689,
+      -0.257373007925008,
+      -0.17157873617869576,
+      -0.5820549318596712,
+      -0.051051515342219425,
+      0.08249499351422646,
+      0.5624601534143484,
+      -0.10303987481127491,
+      0.15033774699941863,
+      -0.5745389903062827,
+      0.23351039478478441,
+      -1.1902195091811327,
+      -0.908353549224439,
+      -0.6461948833956079,
+      -1.393386540097237,
+      -1.0614970718967256,
+      0.30286459932527054,
+      0.3651559686763726,
+      1.1301843929017004
+    ],
+    "std": [
+      0.11613444573720885,
+      0.11901790941228217,
+      0.12423960101251118,
+      2.0003104261737987,
+      0.45447134679198004,
+      0.7816114912726152,
+      0.11129045505686999,
+      0.12915989025569613,
+      0.3554567667529193,
+      0.1287770117110695,
+      0.08437069436241484,
+      0.3662595753207142,
+      0.13719174278620397,
+      0.10683695378613353,
+      0.26563961902562566,
+      0.1614757901944507,
+      0.12472634019043978,
+      0.4459152446405047,
+      0.07304477943041221,
+      0.08736058807876065,
+      0.3190953696852229,
+      0.07287199652137899,
+      0.06397730661800535,
+      0.2664615892181941,
+      0.33533353619474193,
+      0.21176998771110275,
+      0.3620948338112789,
+      0.35230210774422543,
+      0.20843244550288792,
+      0.22361627270253653,
+      0.22739803761052255,
+      0.18855208037939816,
+      0.34182438278954985,
+      0.12194055274979114,
+      0.12564610143293212,
+      0.4347866518877402,
+      0.0895840032117932,
+      0.1557079447450095,
+      0.3181528205878221,
+      0.09813314865033251,
+      0.08801367369557361,
+      0.28670638044605756,
+      0.2929319771897599,
+      0.20330135882077022,
+      0.15333415522323987,
+      0.17407193233130555,
+      0.19461975554089886,
+      0.32521003963028144,
+      0.22419783328653944,
+      0.2856585962011231,
+      0.28531485693053504,
+      0.6395046944136171,
+      0.2581457810827793,
+      0.6770729800021699,
+      0.32191819394761656,
+      0.17693328647166634,
+      0.5711142959757236,
+      0.23986195189121756,
+      0.16371960905341892,
+      0.17578311793753235,
+      0.15744902675287897
+    ]
+  },
+  "action_right": {
+    "mean": [
+      -9.183413654731516e-05,
+      -0.0004571724624278521,
+      0.00036719984809390675,
+      -0.00034841353207357736,
+      0.0002609988965807757,
+      -0.00011327695069714617,
+      0.001417289039899506,
+      -0.20218176833804377,
+      0.4881301482481246,
+      0.14466420684578885,
+      -0.02055669114716166,
+      0.7651701713220371,
+      -0.0619237971459009,
+      0.12226079105293888,
+      0.26130075212237247,
+      -0.1806225022246296,
+      0.008218724116626499,
+      0.8039644050750685,
+      -0.18134867853171585,
+      -0.052555009159906345,
+      0.6154517496548818,
+      0.019421993022190682,
+      0.08211448619373866,
+      0.4802323569413772,
+      -0.2927226007138426,
+      0.5923700304028582,
+      0.6205759342117255,
+      -0.2515955699103106,
+      0.042618912415462076,
+      0.48433096342576415,
+      -0.33498599322011113,
+      0.1567261381224921,
+      0.29128616993263456,
+      0.002590782578856707,
+      0.1963989576489573,
+      0.8355420400922449,
+      -0.3601471445064349,
+      0.18183089067360303,
+      0.4668307340842131,
+      -0.15808449133406852,
+      0.15278912307093284,
+      0.5630418833448485,
+      0.9409358006882846,
+      -0.2575103383389611,
+      -0.17167577461292732,
+      -0.5821898438191407,
+      -0.051104438656083856,
+      0.0823863891010596,
+      0.5625648844622815,
+      -0.10288571771532973,
+      0.15040164745539095
+    ],
+    "std": [
+      0.007176547371510501,
+      0.007322513109397528,
+      0.007113076962986769,
+      0.038237839376925195,
+      0.04096481245854053,
+      0.04100905449992395,
+      0.11131317098602565,
+      0.1291847660339212,
+      0.3554905218962282,
+      0.1287747108411097,
+      0.08437966711877376,
+      0.3663123509682592,
+      0.13721371515626715,
+      0.1068594576234084,
+      0.2657702967257421,
+      0.1615031274249978,
+      0.12472932981401375,
+      0.44596150225826314,
+      0.0730684114259582,
+      0.08739727255221826,
+      0.31921839859373785,
+      0.07287343116568247,
+      0.06399901491867792,
+      0.2665651762569098,
+      0.3354871477636308,
+      0.21180522510052424,
+      0.3621549257811261,
+      0.3524644945639798,
+      0.20851868839392335,
+      0.22370211394987788,
+      0.22751559151704984,
+      0.1886216707243024,
+      0.34189811196929315,
+      0.12198624275798856,
+      0.12562642132107327,
+      0.4348974375473972,
+      0.08959453757611424,
+      0.15574468223655724,
+      0.31824911600176237,
+      0.09815904234327474,
+      0.08802379060051459,
+      0.28676173641830466,
+      0.29304320335731693,
+      0.20324969302436036,
+      0.15336124234578655,
+      0.17407549505733239,
+      0.19463239011417244,
+      0.32515766963030956,
+      0.2242245515909087,
+      0.2857767125669773,
+      0.2853074008480897
+    ]
+  }
+}

vitra-vla-3b.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f57dfd4451d68f41accab8a06bb388d6fdc2cce87ee8b31fafa7c8f01d83796
+size 15067578877