VisualCloze/para/depth/log.txt

[2025-07-28 03:39:45] Experiment directory created at /nvme-data/Komal/documents/results/VisualCloze/para/depth
[2025-07-28 03:39:46] Downloaded model to /nvme-data/Komal/huggingface/hub/models--Shitao--OmniGen-v1/snapshots/58e249c7c7634423c0ba41c34a774af79aa87889
[2025-07-28 03:39:46] Downloaded model to /nvme-data/Komal/huggingface/hub/models--Shitao--OmniGen-v1/snapshots/58e249c7c7634423c0ba41c34a774af79aa87889
[2025-07-28 03:40:31] Trainable parameters: 3,145,728 (para)
[2025-07-28 03:40:31] Total parameters in the model: 3,762,072,592 (para)
[2025-07-28 03:40:41] Dataset contains 205,841
[2025-07-28 03:40:44] Training for 2000 epochs...
[2025-07-28 03:40:44] Beginning epoch 0...
[2025-07-28 03:40:57] EFFICIENCY_METRICS: {"epoch": 0, "step": 1, "loss": 0.3588107228279114, "memory_gb": 7.7092108726501465, "step_time_ms": 12695.839881896973, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:40:57] (step=0000001) Train Loss: 0.3116, Train Steps/Sec: 0.07, Epoch: 1.9432568985619897e-05, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:41:09] EFFICIENCY_METRICS: {"epoch": 0, "step": 2, "loss": 0.24570123851299286, "memory_gb": 7.710222244262695, "step_time_ms": 10826.501369476318, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:41:09] (step=0000002) Train Loss: 0.2334, Train Steps/Sec: 0.09, Epoch: 3.8865137971239795e-05, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:41:21] EFFICIENCY_METRICS: {"epoch": 0, "step": 3, "loss": 0.2485000193119049, "memory_gb": 7.709211349487305, "step_time_ms": 10868.380546569824, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:41:21] (step=0000003) Train Loss: 0.2753, Train Steps/Sec: 0.09, Epoch: 5.82977069568597e-05, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:41:33] EFFICIENCY_METRICS: {"epoch": 0, "step": 4, "loss": 0.29748108983039856, "memory_gb": 7.710222244262695, "step_time_ms": 10847.830533981323, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:41:33] (step=0000004) Train Loss: 0.2768, Train Steps/Sec: 0.08, Epoch: 7.773027594247959e-05, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:41:44] EFFICIENCY_METRICS: {"epoch": 0, "step": 5, "loss": 0.3283542990684509, "memory_gb": 7.703290939331055, "step_time_ms": 10684.507846832275, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:41:44] (step=0000005) Train Loss: 0.3362, Train Steps/Sec: 0.09, Epoch: 9.71628449280995e-05, LR: 0.001, Memory: 7.70GB, Params: 3,145,728
[2025-07-28 03:41:56] EFFICIENCY_METRICS: {"epoch": 0, "step": 6, "loss": 0.2764468491077423, "memory_gb": 7.710222244262695, "step_time_ms": 10916.002035140991, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:41:56] (step=0000006) Train Loss: 0.3372, Train Steps/Sec: 0.09, Epoch: 0.0001165954139137194, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:42:08] EFFICIENCY_METRICS: {"epoch": 0, "step": 7, "loss": 0.2880154550075531, "memory_gb": 7.709211349487305, "step_time_ms": 10734.980583190918, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:42:08] (step=0000007) Train Loss: 0.3058, Train Steps/Sec: 0.09, Epoch: 0.0001360279828993393, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:42:20] EFFICIENCY_METRICS: {"epoch": 0, "step": 8, "loss": 0.23556765913963318, "memory_gb": 7.709367752075195, "step_time_ms": 10774.001359939575, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:42:20] (step=0000008) Train Loss: 0.2780, Train Steps/Sec: 0.08, Epoch: 0.00015546055188495918, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:42:31] EFFICIENCY_METRICS: {"epoch": 0, "step": 9, "loss": 0.25627702474594116, "memory_gb": 7.709211349487305, "step_time_ms": 10786.931991577148, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:42:31] (step=0000009) Train Loss: 0.3002, Train Steps/Sec: 0.08, Epoch: 0.00017489312087057908, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:42:43] EFFICIENCY_METRICS: {"epoch": 0, "step": 10, "loss": 0.13957127928733826, "memory_gb": 7.709367752075195, "step_time_ms": 10750.224828720093, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:42:43] (step=0000010) Train Loss: 0.1853, Train Steps/Sec: 0.09, Epoch: 0.000194325689856199, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:42:55] EFFICIENCY_METRICS: {"epoch": 0, "step": 11, "loss": 0.21578535437583923, "memory_gb": 7.709211349487305, "step_time_ms": 10592.220306396484, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:42:55] (step=0000011) Train Loss: 0.2714, Train Steps/Sec: 0.08, Epoch: 0.0002137582588418189, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:43:05] EFFICIENCY_METRICS: {"epoch": 0, "step": 12, "loss": 0.2423926740884781, "memory_gb": 7.709367752075195, "step_time_ms": 9401.668548583984, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:43:05] (step=0000012) Train Loss: 0.2496, Train Steps/Sec: 0.10, Epoch: 0.0002331908278274388, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:43:16] EFFICIENCY_METRICS: {"epoch": 0, "step": 13, "loss": 0.2849031090736389, "memory_gb": 7.709211349487305, "step_time_ms": 9934.309959411621, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:43:16] (step=0000013) Train Loss: 0.3133, Train Steps/Sec: 0.09, Epoch: 0.0002526233968130587, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:43:28] EFFICIENCY_METRICS: {"epoch": 0, "step": 14, "loss": 0.19880621135234833, "memory_gb": 7.709367752075195, "step_time_ms": 10697.36933708191, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:43:28] (step=0000014) Train Loss: 0.1959, Train Steps/Sec: 0.09, Epoch: 0.0002720559657986786, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:43:40] EFFICIENCY_METRICS: {"epoch": 0, "step": 15, "loss": 0.24679075181484222, "memory_gb": 7.709211349487305, "step_time_ms": 10716.626167297363, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:43:40] (step=0000015) Train Loss: 0.2577, Train Steps/Sec: 0.08, Epoch: 0.0002914885347842985, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:43:51] EFFICIENCY_METRICS: {"epoch": 0, "step": 16, "loss": 0.24902749061584473, "memory_gb": 7.709367752075195, "step_time_ms": 10611.615419387817, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:43:51] (step=0000016) Train Loss: 0.2786, Train Steps/Sec: 0.09, Epoch: 0.00031092110376991836, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:44:03] EFFICIENCY_METRICS: {"epoch": 0, "step": 17, "loss": 0.26482874155044556, "memory_gb": 7.709211349487305, "step_time_ms": 10648.298740386963, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:44:03] (step=0000017) Train Loss: 0.2505, Train Steps/Sec: 0.09, Epoch: 0.00033035367275553826, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:44:15] EFFICIENCY_METRICS: {"epoch": 0, "step": 18, "loss": 0.2249012291431427, "memory_gb": 7.709367752075195, "step_time_ms": 10705.644845962524, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:44:15] (step=0000018) Train Loss: 0.2237, Train Steps/Sec: 0.08, Epoch: 0.00034978624174115817, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:44:27] EFFICIENCY_METRICS: {"epoch": 0, "step": 19, "loss": 0.1946757733821869, "memory_gb": 7.709211349487305, "step_time_ms": 10714.349031448364, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:44:27] (step=0000019) Train Loss: 0.2289, Train Steps/Sec: 0.09, Epoch: 0.00036921881072677807, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:44:38] EFFICIENCY_METRICS: {"epoch": 0, "step": 20, "loss": 0.16707903146743774, "memory_gb": 7.709367752075195, "step_time_ms": 10697.560787200928, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:44:38] (step=0000020) Train Loss: 0.2529, Train Steps/Sec: 0.09, Epoch: 0.000388651379712398, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:44:50] EFFICIENCY_METRICS: {"epoch": 0, "step": 21, "loss": 0.3534172773361206, "memory_gb": 7.709211349487305, "step_time_ms": 10671.65231704712, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:44:50] (step=0000021) Train Loss: 0.3348, Train Steps/Sec: 0.09, Epoch: 0.0004080839486980179, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:45:02] EFFICIENCY_METRICS: {"epoch": 0, "step": 22, "loss": 0.2638294994831085, "memory_gb": 7.709367752075195, "step_time_ms": 10747.231721878052, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:45:02] (step=0000022) Train Loss: 0.2652, Train Steps/Sec: 0.08, Epoch: 0.0004275165176836378, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:45:14] EFFICIENCY_METRICS: {"epoch": 0, "step": 23, "loss": 0.2914794683456421, "memory_gb": 7.709211349487305, "step_time_ms": 10627.845048904419, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:45:14] (step=0000023) Train Loss: 0.3131, Train Steps/Sec: 0.09, Epoch: 0.0004469490866692577, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:45:25] EFFICIENCY_METRICS: {"epoch": 0, "step": 24, "loss": 0.2621877193450928, "memory_gb": 7.709367752075195, "step_time_ms": 10651.673555374146, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:45:25] (step=0000024) Train Loss: 0.2670, Train Steps/Sec: 0.09, Epoch: 0.0004663816556548776, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:45:37] EFFICIENCY_METRICS: {"epoch": 0, "step": 25, "loss": 0.27799099683761597, "memory_gb": 7.709211349487305, "step_time_ms": 10689.085483551025, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:45:37] (step=0000025) Train Loss: 0.2413, Train Steps/Sec: 0.08, Epoch: 0.0004858142246404975, LR: 0.001, Memory: 7.71GB, Params: 3,145,728
[2025-07-28 03:45:49] EFFICIENCY_METRICS: {"epoch": 0, "step": 26, "loss": 0.19166377186775208, "memory_gb": 7.709367752075195, "step_time_ms": 10644.321203231812, "trainable_params": 3145728, "method": "para"}
[2025-07-28 03:45:49] (step=0000026) Train Loss: 0.2635, Train Steps/Sec: 0.09, Epoch: 0.0005052467936261174, LR: 0.001, Memory: 7.71GB, Params: 3,145,728