Upload experiment directory test_conformer_10-15_07-53

.gitattributes

@@ -33,3 +33,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+test_conformer_10-15_07-53/_nemo_model_part_0.nemo filter=lfs diff=lfs merge=lfs -text
+test_conformer_10-15_07-53/_nemo_model_part_1.nemo filter=lfs diff=lfs merge=lfs -text
+test_conformer_10-15_07-53/_nemo_model_part_2.nemo filter=lfs diff=lfs merge=lfs -text
+test_conformer_10-15_07-53/_nemo_model_part_3.nemo filter=lfs diff=lfs merge=lfs -text
+test_conformer_10-15_07-53/_nemo_model_part_4.nemo filter=lfs diff=lfs merge=lfs -text
+test_conformer_10-15_07-53/_nemo_model_part_5.nemo filter=lfs diff=lfs merge=lfs -text
+test_conformer_10-15_07-53/nemo_model_avg.nemo filter=lfs diff=lfs merge=lfs -text
+test_conformer_10-15_07-53/training_process_4.png filter=lfs diff=lfs merge=lfs -text
+test_conformer_10-15_07-53/training_process_5.png filter=lfs diff=lfs merge=lfs -text

test_conformer_10-15_07-53/_dev_mf.json

@@ -0,0 +1,9 @@
+{"audio_filepath": "/kaggle/input/annot-ds/annot-ds/wavs/97_0259.wav", "duration": 25.433, "text": "1 th ì 1 tr ướ c 1 đ â y 1 th ì 1 kh ô ng 1 c ó 1 đ ườ ng 1 đ i 1 l ạ i 1 ấ y 1 th ì 1 h ọ c 1 s i nh 1 k iể u 1 đ i 1 b ộ 1 c ó 1 h ô m 1 th ì 1 h ọ c 1 s i nh 1 đ i 1 đ ế n 1 n ửa 1 đ ườ ng 1 đ ã 1 l ẩ n 1 tr ố n 1 đ i 1 v à o 1 c ạ nh 1 đ ườ ng 1 l ề 1 đ ườ ng 1 h oặ c 1 l à 1 đ ế n 1 tr ưa 1 tr ưa 1 r ồ i 1 l ạ i 1 v ề 1 x o ng 1 r ồ i 1 t ừ 1 l ú c 1 c ó 1 đ ườ ng 1 r ồ i 1 th ì 1 b ố 1 m ẹ 1 m ườ i 1 m ấ y 1 tr ở 1 l ạ i 1 đ â y 1 th ì 1 c ũ ng 1 gi a 1 đ ì nh 1 c ũ ng 1 c ó 1 m ộ t 1 c á i 1 x e 1 x o ng 1 r ồ i 1 l ú c 1 k ia 1 đ ưa 1 c o n 1 đ i 1 h ọ c 1 đ ưa 1 đ ế n 1 n ơ i 1 v ề 1 đ ế n 1 ch ố n"}
+{"audio_filepath": "/kaggle/input/annot-ds/annot-ds/wavs/70_0247.wav", "duration": 10.4163125, "text": "0 b ê n 0 c ô ng 0 a n 0 th ị 0 tr ấ n 0 th ì 0 b ướ c 0 t uyê n 0 tr uyề n 0 r ấ t 0 l à 0 t ố t 0 r ấ t 0 l à 0 t í ch 0 c ự c 0 t uyê n 0 tr uyề n 0 ch o 0 b à 0 c o n 0 đ ế n 0 t ừ ng 0 h ộ 0 nh à 0 c ũ ng 0 th ô ng 0 b á o 0 ch o 0 b à 0 c o n 0 đ ế n 0 đ â y 0 l à m"}
+{"audio_filepath": "/kaggle/input/annot-ds/annot-ds/wavs/86_0322.wav", "duration": 22.906, "text": "0 g â y 0 h ạ i 0 nh iề u 0 h ơ n 0 s o 0 v ớ i 0 v ườ n 0 đ ượ c 0 th ô ng 0 th oá ng 0 th ưa 0 b à 0 c o n 0 th ì 0 đ ố i 0 v ớ i 0 v ề 0 s i nh 0 l ý 0 c â y 0 đ iề u 0 l à 0 c â y 0 t ậ p 0 tr u ng 0 r a 0 h oa 0 ở 0 đ ầ u 0 c à nh 0 đ ầ u 0 t á n 0 n ê n 0 c á c 0 c á i 0 c à nh 0 tr o ng 0 t á n 0 th ì 0 n ó 0 s ẽ 0 kh ô ng 0 ch o 0 h oa 0 h iệ u 0 q uả 0 n ê n 0 b à 0 c o n 0 t iế n 0 h àn nh 0 c á c 0 c ắ t 0 t ỉa 0 c á c 0 c à nh 0 tr o ng 0 t á n 0 v à 0 ch ă m 0 s ó c 0 v à"}
+{"audio_filepath": "/kaggle/input/annot-ds/annot-ds/wavs/60_0256.wav", "duration": 24.85, "text": "0 s ự 0 ph ố i 0 h ợ p 0 v ớ i 0 c á c 0 ph ò ng 0 b a n 0 ch uyê n 0 m ô n 0 v í 0 d ụ 0 ph ò ng 0 t à i 0 ng uyê n 0 m ô i 0 tr ườ ng 0 ph ò ng 0 k i nh 0 t ế 0 h ạ 0 t ầ ng 0 ph ò ng 0 n ôg ng 0 ngh iệ p 0 c á c 0 gi á m 0 kh ả o 0 đ ã 0 c ó 0 k i nh 0 ngh iệ m 0 c á c 0 a nh 0 ch ị 0 đ ã 0 c ó 0 k i nh 0 ngh iệ m 0 c ũ ng 0 nh ư 0 đ ạ i 0 h ọ c 0 l ạ c 0 h ồ ng 0 c ó 0 h a i 0 th ầ y 0 ở 0 đ ạ i 0 h ọ c 0 l ạ c 0 h ồ ng 0 v ề 0 c ù ng 0 ch ấ m"}
+{"audio_filepath": "/kaggle/input/annot-ds/annot-ds/wavs/64_0265.wav", "duration": 10.342, "text": "2 r ồ i 2 h ỗ 2 tr ợ 2 v ề 2 ứ ng 2 d ụ ng 2 c ô ng 2 ngh ệ 2 c a o 2 h ỗ 2 tr ợ 2 v ề 2 ch i 2 ph í 2 x â y 2 d ự ng 2 nh à 2 x ưở ng 2 m á y 2 m ó c 2 th iế t 2 b ị 2 nh ư ng 2 m à 2 c ũ ng 2 kh ô ng 2 q uá 2 h a i 2 t ỷ 2 đ ồ ng 2 ch o 2 m ộ t 2 d ự 2 á n"}
+{"audio_filepath": "/kaggle/input/annot-ds/annot-ds/wavs/27_0257.wav", "duration": 22.7170625, "text": "1 s a u 1 kh i 1 e m 1 t ố t 1 ngh iệ p 1 x o ng 1 c ấ p 1 b a 1 x o ng 1 th ì 1 c ó 1 r ấ t 1 nh iề u 1 l ựa 1 ch ọ n 1 ng à nh 1 ngh ề 1 m à 1 ng ườ i 1 th â n 1 tr o ng 1 gi a 1 đ ì nh 1 đ ã 1 gi ớ i 1 th iệ u 1 ch o 1 e m 1 nh ư ng 1 s a u 1 đ ó 1 e m 1 v ẫ n 1 q uyế t 1 đ ị nh 1 ch ọ n 1 t ạ i 1 tr ườ ng 1 c a o 1 đ ẳ ng 1 ngh ề 1 n à y 1 th ì 1 e m 1 c ả m 1 th ấ y 1 e m 1 nh ậ n 1 đ ị nh 1 đ ượ c 1 h iệ n 1 t ạ i 1 ng à nh 1 c ô ng 1 ngh ệ 1 ô 1 t ô 1 c ũ ng 1 đ a ng 1 r ấ t 1 c ầ n 1 nh â n 1 c ô ng 1 v à 1 l a o 1 đ ộ ng 1 th ì 1 e m 1 q uyế t 1 đ ị nh 1 h ọ c 1 đ ể 1 đ á p 1 ứ ng 1 nh u 1 c ầ u 1 c ủa 1 th ị 1 tr ườ ng 1 s a u 1 đ ó 1 m o ng 1 m uố n 1 l à 1 s ẽ 1 t ừ 1 c ô ng 1 v iệ c 1 n à y 1 c ó 1 m ộ t 1 m ứ c 1 l ươ ng 1 ổ n 1 đ ị nh"}
+{"audio_filepath": "/kaggle/input/annot-ds/annot-ds/wavs/17_0289.wav", "duration": 12.551, "text": "7 c à ng 7 th a m 7 gi a 7 c á c 7 ph o ng 7 tr à o 7 th ể 7 d ụ c 7 th ể 7 th a o 7 v ă n 7 h óa 7 ngh ệ 7 th uậ t 7 b a o 7 nh iê u 7 th ì 7 l à 7 c à ng 7 t ố t 7 ch o 7 ng ườ i 7 c a o 7 t uổ i 7 b ấ y 7 nh iê u 7 t ô i 7 c ả m 7 th ấ y 7 l à 7 s ứ c 7 kh ỏe 7 c ủa 7 t uổ i 7 gi à 7 đ ượ c 7 n âg ng 7 l ê n 7 r ấ t 7 l à 7 nh iề u 7 nh ư ng 7 m à 7 t i nh 7 th ầ n 7 c ả m 7 th ấ y 7 l à 7 s ả ng 7 kh oá i 7 v u i 7 t ươ i"}
+{"audio_filepath": "/kaggle/input/annot-ds/annot-ds/wavs/99_0277.wav", "duration": 27.7096875, "text": "1 t ạ m 1 đ ế n 1 gi ờ 1 th ì 1 c ơ 1 b ả n 1 th ì 1 n ó 1 h ơ n 1 ng à y 1 x ưa 1 m ộ t 1 ch ú t 1 nh ư ng 1 m à 1 v ề 1 c á i 1 đ ố t 1 v ề 1 c á i 1 b ã 1 r á c 1 th ả i 1 v ẫ n 1 đ ổ 1 ở 1 đ â y 1 n ó 1 v ẫ n 1 ch á y 1 â m 1 ỉ 1 ch o 1 n ê n 1 ng ườ i 1 d â n 1 ch ú ng 1 t ô i 1 v ẫ n 1 ph ả i 1 h ứ ng 1 ch ị u 1 c á i 1 kh ó i 1 kh ô ng 1 kh í 1 c ủa 1 b ã i 1 r á c 1 l ê n 1 m o ng 1 m uố n 1 ch ú ng 1 t ô i 1 b â y 1 gi ờ 1 th ì 1 c ứ 1 đ ề 1 x uấ t 1 c o i 1 nh ư 1 l à 1 c á n 1 b ộ 1 c o i 1 nh ư 1 l à 1 x ã 1 h oà n 1 s ơ n 1 v à 1 t iê n 1 d u 1 v à 1 c á n 1 b ộ 1 t ỉ nh 1 b ắ c 1 n ih nh 1 c o i 1 nh ư 1 l à 1 s ớ m 1 d i 1 ch uyể n 1 c á i 1 b ã i 1 r á c 1 n à y 1 ch o 1 ng ườ i 1 d â n 1 ch ú ng 1 t ô i 1 đ ượ c 1 ch o 1 c á c 1 ch á u 1 đ i 1 h ọ c 1 m à 1 m ô i 1 tr ườ ng 1 n ó 1 tr o ng 1 s ạ ch 1 h ơ n"}
+{"audio_filepath": "/kaggle/input/annot-ds/annot-ds/wavs/36_0284.wav", "duration": 19.465, "text": "1 l iê n 1 t iế p 1 x ả y 1 r a 1 c á c 1 v ụ 1 r ấ t 1 ngh iê m 1 tr ọ ng 1 tr ướ c 1 c á i 1 t ì nh 1 h ìn nh 1 tr ê n 1 ph ò ng 1 c ả nh 1 s á t 1 h ìn nh 1 s ự 1 đ ã 1 th a m 1 m ư u 1 ch o 1 gi á m 1 đ ố c 1 c ô ng 1 a n 1 t ỉ nh 1 m ộ t 1 k ế 1 h oạc ch 1 h uy 1 đ ộ ng 1 l ự c 1 l ượ ng 1 c á n 1 b ộ 1 ph ươ ng 1 t iệ n 1 v à 1 th ự c 1 h iệ n 1 c á c 1 c á i 1 gi ả i 1 ph á p 1 ph ò ng 1 ng ừa 1 v à 1 đ ấ u 1 tr a nh"}

test_conformer_10-15_07-53/_nemo_model_part_0.nemo

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94b83f93d36ffce54259af9ca19d5005af3a914b061b854f51786d647b999eba
+size 52439040

test_conformer_10-15_07-53/_nemo_model_part_1.nemo

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9da266d37a540b54064b5fc54d903b0dec9de659fb1c19df62dcd042e9f0d897
+size 52439040

test_conformer_10-15_07-53/_nemo_model_part_2.nemo

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52212e55c2fc58ea0cb4ca22d619b674b091fdaa44f4f7506a0aae1760769434
+size 52439040

test_conformer_10-15_07-53/_nemo_model_part_3.nemo

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:080979828252ae0db0d7ff74b0a49c985db2b20fdff8a7d01bc663975c76c35f
+size 52439040

test_conformer_10-15_07-53/_nemo_model_part_4.nemo

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d253fe1024f85875cb2c7deec4ee9873003e8dfb99870eb18cbf5d14b717fdb0
+size 52439040

test_conformer_10-15_07-53/_nemo_model_part_5.nemo

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe47b74fe87434de406c85dded2b82a1c82a7909adeadc120912242a54be4b72
+size 52439040

test_conformer_10-15_07-53/args_trainer.txt

@@ -0,0 +1 @@
+{'precision': 'bf16', 'devices': 1, 'num_nodes': 1, 'accelerator': 'gpu', 'strategy': 'auto', 'max_epochs': 1000, 'accumulate_grad_batches': 1, 'gradient_clip_val': 0.0, 'log_every_n_steps': 100, 'val_check_interval': 1.0, 'enable_progress_bar': False, 'num_sanity_val_steps': 0, 'check_val_every_n_epoch': 1, 'sync_batchnorm': True, 'benchmark': False, 'enable_checkpointing': True, 'max_time': '00:00:02:00', 'callbacks': [<v2_run.Conformer_nemo.utils.utils.LossLogger object at 0x7f7cf18a4f90>, <lightning.pytorch.callbacks.early_stopping.EarlyStopping object at 0x7f7cf176ee50>, <lightning.pytorch.callbacks.model_checkpoint.ModelCheckpoint object at 0x7f7cf191d4d0>]}

test_conformer_10-15_07-53/ckpts/epochepoch=0-val_wer=0.8375.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25da216a1443f5231f468571a98047b5efc58c601304b460c712e75aa47d81b9
+size 157153333

test_conformer_10-15_07-53/ckpts/epochepoch=0-val_wer=0.9905.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:020ffb258daee4ab1bbd14fe240c8f3f68da56338675b3023b54c00603af5804
+size 157153333

test_conformer_10-15_07-53/ckpts/epochepoch=0-val_wer=0.9911.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5e4c938bf922887732a1c312baeded9d9750c9d76cd3cac106f4a311d4c0403
+size 157153333

test_conformer_10-15_07-53/ckpts/epochepoch=0-val_wer=0.9945.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:798505303504bb1a5668b6bcfef485c62f288dc07080524dd7b71fd73df31a52
+size 157153333

test_conformer_10-15_07-53/ckpts/epochepoch=0-val_wer=0.9984.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:450dc49a39185df4d25a7e756e4cdf0a104034ef74a5350a195f0f4755abd63f
+size 157153333

test_conformer_10-15_07-53/ckpts/epochepoch=1-val_wer=0.9905.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c78da6cae0e4c580d2dcde38c94ca49b62640ead5076668c36f867e1beb1e423
+size 157153716

test_conformer_10-15_07-53/ckpts/epochepoch=1-val_wer=0.9911.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8920815aadc771aaf0b60305895413c79b57675ce3145441c020f2ede60c32ba
+size 157153716

test_conformer_10-15_07-53/ckpts/epochepoch=1-val_wer=0.9933.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27caca5d6b80f10d7e7d8b73798b4e14a6287bd1fb24279be5a5f45b1b57daf3
+size 157153716

test_conformer_10-15_07-53/ckpts/epochepoch=1-val_wer=0.9983.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d29d411c127dbfee0a69d60f99a957a3b2626afe02258277b51fe43e82210f4
+size 157153716

test_conformer_10-15_07-53/ckpts/epochepoch=1-val_wer=0.9984.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5166a6863bac5a6823dd56a3f5bb32bfdbf2f9893acc7b5ddf5c8c2d6234444d
+size 157153716

test_conformer_10-15_07-53/ckpts/epochepoch=1-val_wer=0.9988.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c41cf1694aac103149a9b363f8d85fd021a2edc39c14bd042db98b4da69a86f
+size 157153716

test_conformer_10-15_07-53/ckpts/epochepoch=2-val_wer=0.9899.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e6acbe811b61314c231b6fcdd0e88cd18aeffee9807368fe1dfedaaf4e55cd7
+size 157154099

test_conformer_10-15_07-53/ckpts/epochepoch=2-val_wer=0.9911.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a99e510a22ecef7566d1c5219e1971de107625e56ee6646e366c7c13ab84efb2
+size 157154099

test_conformer_10-15_07-53/ckpts/epochepoch=2-val_wer=0.9933.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea0f350f4885b2e0dee5d725ce504bbae55075ad4853abca5566aa69ba865505
+size 157154099

test_conformer_10-15_07-53/ckpts/epochepoch=2-val_wer=0.9983.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c91812b643ef97486dbded4105feb0f6c7b4d8e3b6263204ea0bc0c2274db6ce
+size 157154099

test_conformer_10-15_07-53/ckpts/epochepoch=2-val_wer=0.9984.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d415fed79b55a0f7446fb1388b52761eb065ae8d90a3458a40ef7832aa5e6387
+size 157154099

test_conformer_10-15_07-53/ckpts/epochepoch=2-val_wer=0.9988.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e475e27b158528dad0d7d807d05c986d6708f58d121342bf7fa9a82f9ada763
+size 157154099

test_conformer_10-15_07-53/ckpts/epochepoch=3-val_wer=0.9879.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:860702c6a021cbfbbab42198af4c844f45c598daa23cd641c261aa1308a4d2a7
+size 157154418

test_conformer_10-15_07-53/ckpts/epochepoch=3-val_wer=0.9911.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c67cfff55ba9cbfdc39b025fa72e9a795412ed5419ba800934c0037750e49c9f
+size 157154418

test_conformer_10-15_07-53/ckpts/epochepoch=3-val_wer=0.9933.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5870e3b4e7d8a0f324364cc880f79ee0ceabb260a7383765cc2e0dc425999f33
+size 157154418

test_conformer_10-15_07-53/ckpts/epochepoch=3-val_wer=0.9983.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13b013f7e5d6e2ebaeee7f0f167113b73e55b5bd048be3a548886c52aac3f385
+size 157154418

test_conformer_10-15_07-53/ckpts/epochepoch=3-val_wer=0.9984.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:567482c617230158a1e3714bef8a0fd00d1e6a382c1080b16f2c6db25a6f52b9
+size 157154418

test_conformer_10-15_07-53/ckpts/epochepoch=3-val_wer=0.9988.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d13454913774a1e4b5df920e96bdcebf79f57564a079bc8a6ae5c34e4b127f4
+size 157154418

test_conformer_10-15_07-53/ckpts/epochepoch=4-val_wer=0.9905.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c72589eb6a294fe408f729b1ec34bac7a3e66bd2147c7eb66c44faac281e92c
+size 157154482

test_conformer_10-15_07-53/ckpts/epochepoch=4-val_wer=0.9911.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11d3dab7f84326bbfcaa6a7e624abd08461bf403cd1d34179dd9c48c6c1e9bfc
+size 157154482

test_conformer_10-15_07-53/ckpts/epochepoch=4-val_wer=0.9926.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1871ee4d27078cd2a67b77e877be19649994e9ece35816116b6a9852a403ad65
+size 157154482

test_conformer_10-15_07-53/ckpts/epochepoch=4-val_wer=0.9933.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a754ecb11262a6b295d1dd995ac4ec5726418e8478d02fc9420591305f61f10
+size 157154482

test_conformer_10-15_07-53/ckpts/epochepoch=4-val_wer=0.9983.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03848225c6243cbf8434fc7e1ca948f445db305c2c2bd2fc19157850eb37e847
+size 157154482

test_conformer_10-15_07-53/ckpts/epochepoch=4-val_wer=0.9984.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2ad497ee2257cf6cd3b3dd8783d3a8563b95d699d7ca27991c5f606b89ab23c
+size 157154482

test_conformer_10-15_07-53/ckpts/epochepoch=5-val_wer=0.9922.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1e4f0c67969218a351d0ba4b6ad924445132c150861f2214361e1321af47fba
+size 157154482

test_conformer_10-15_07-53/code-folder/configs/conformer.yaml

@@ -0,0 +1,265 @@
+# It contains the default values for training a Conformer-CTC ASR model, large size (~120M) with CTC loss and sub-word encoding.
+
+# Architecture and training config:
+# Here are the recommended configs for different variants of Conformer-CTC, other parameters are the same as in this config file.
+#! One extra layer (compared to original paper) is added to the #medium and #large variants to compensate for replacing the #LSTM decoder with a linear one.
+#
+#  +--------------+---------+---------+----------+------------------+------------+-----+
+#  | Model        | d_model | n_heads | n_layers | conv_kernel_size | time_masks | lr  |
+#  +==============+=========+========+===========+==================+============+=====+
+#  | Small   (13M)|   176   |    4   |    16     |        31        |     5      | 5.0 |
+#  +--------------+---------+--------+-----------+------------------+------------+-----+
+#  | Medium  (30M)|   256   |    4   |    18     |        31        |     5      | 5.0 |
+#  +--------------+---------+--------+-----------+------------------+------------+-----+
+#  | Large  (121M)|   512   |    8   |    18     |        31        |     10     | 2.0 |
+#  +------------------------+--------+-----------+------------------+------------+-----+
+#  | XLarge (635M)|  1024   |    8   |    24     |         5        |     10     | 6.4 |
+#  +--------------+---------+--------+-----------+------------------+------------+-----+
+#
+# Default learning parameters in this config are set for global batch size of 2K while you may use lower values.
+# To increase the global batch size with limited number of GPUs, you may use higher accumulate_grad_batches.
+# However accumulate_grad_batches is better to be avoided as long as the global batch size is large enough and training is stable.
+
+# You may find more info about Conformer-CTC here: https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/asr/models.html#conformer-ctc
+# Pre-trained models of Conformer-CTC can be found here: https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/asr/results.html
+# The checkpoint of the large model trained on LibriSpeech with this recipe can be found here: https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_large_ls
+
+# We suggest to use trainer.precision=bf16 for GPUs which support it otherwise trainer.precision=16 is recommended.
+# Using bf16 or 16 would make it possible to double the batch size and speedup training/inference. If fp16 is not stable and model diverges after some epochs, you may use fp32.
+# Here are the suggested batch size per GPU for each precision and memory sizes:
+#  +-----------+------------+------------+
+#  | Precision | GPU Memory | Batch Size |
+#  +===========+============+============+
+#  | 32        |    16GB    |     8      |
+#  |           |    32GB    |     16     |
+#  |           |    80GB    |     32     |
+#  +-----------+------------+------------+
+#  | 16 or     |    16GB    |     16     |
+#  | bf16      |    32GB    |     32     |
+#  |           |    80GB    |     64     |
+#  +-----------+------------+------------+
+#! fp16 >> 32 --> batch_size = 16, small size
+# Note:  They are based on the assumption of max_duration of 20. If you have longer or shorter max_duration, then batch sizes may need to get updated accordingly.
+
+name: "Conformer-CTC-BPE"
+train_path: "v2_prepdata/out/train_300.json"
+dev_path: "v2_prepdata/out/test_10.json"
+test_path: "v2_prepdata/out/test_300.json"
+vocab_dir: "v2_prepdata/out/" #* correct
+full_val_path: "v2_prepdata/out/dev_300.json"
+
+train_ds_batch_size: 16 #! batch size for small
+gen_ds_batch_size: 16
+ds_max_duration: 25.5 #! Will be replaced
+
+# Model parameters
+d_model: 176
+n_heads: 4
+n_layers: 16
+conv_kernel_size: 31
+aug_time_masks: 5 # Change from 10 to faster smooth optimize, 2 in fast conformer
+# Optimize
+lr: 5.0 #! Select lr and decay = 0.0, sched, warm_step
+weight_decay: 0.0
+warmup_steps: 10000
+use_cer: true #! Change metrics
+
+model:
+  sample_rate: 16000
+  log_prediction: false #! enables logging sample predictions in the output during training
+  ctc_reduction: "mean_batch" # "mean_volume" in fast conformer
+  skip_nan_grad: false
+
+  wer:
+    use_cer: ${use_cer}
+  use_cer: ${use_cer} # Set to true for Character Error Rate instead (default: false)
+
+  train_ds:
+    manifest_filepath: ${train_path}
+    sample_rate: ${model.sample_rate}
+    batch_size: ${train_ds_batch_size}
+    shuffle: true
+    num_workers: 8
+    pin_memory: true
+    max_duration: ${ds_max_duration} # it is set for LibriSpeech, you may need to update it for your dataset
+    min_duration: 0.1
+    # tarred datasets
+    is_tarred: false
+    tarred_audio_filepaths: null
+    shuffle_n: 2048
+    # bucketing params
+    bucketing_strategy: "synced_randomized" # "fully_randomized" in fast conformer
+    bucketing_batch_size: null
+
+  validation_ds:
+    manifest_filepath: ${dev_path}
+    sample_rate: ${model.sample_rate}
+    batch_size: ${gen_ds_batch_size} # you may increase batch_size if your memory allows
+    shuffle: false
+    use_start_end_token: false
+    num_workers: 8
+    pin_memory: true
+
+  test_ds:
+    manifest_filepath: ${test_path}
+    sample_rate: ${model.sample_rate}
+    batch_size: ${gen_ds_batch_size} # you may increase batch_size if your memory allows
+    shuffle: false
+    use_start_end_token: false
+    num_workers: 8
+    pin_memory: true
+
+  # recommend to SPE Unigram tokenizer with small vocab size of 128 or 256 when using 4x sub-sampling
+  # you may find more detail on how to train a tokenizer at: /scripts/tokenizers/process_asr_text_tokenizer.py
+  tokenizer:
+    dir: ${vocab_dir} # path to directory which contains either tokenizer.model (bpe) or vocab.txt (wpe)
+    type: wpe # Can be either bpe (SentencePiece tokenizer) or wpe (WordPiece tokenizer)
+
+  preprocessor:
+    _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor
+    sample_rate: ${model.sample_rate}
+    normalize: "per_feature"
+    window_size: 0.025
+    window_stride: 0.01
+    window: "hann"
+    features: 80
+    n_fft: 512
+    log: true
+    frame_splicing: 1
+    dither: 0.00001
+    pad_to: 0
+    pad_value: 0.0
+
+  spec_augment:
+    _target_: nemo.collections.asr.modules.SpectrogramAugmentation
+    freq_masks: 2 # set to zero to disable it
+    # you may use lower time_masks for smaller models to have a faster convergence
+    time_masks: ${aug_time_masks} # set to zero to disable it
+    freq_width: 27
+    time_width: 0.05
+
+  encoder:
+    _target_: nemo.collections.asr.modules.ConformerEncoder
+    feat_in: ${model.preprocessor.features}
+    feat_out: -1 # you may set it if you need different output size other than the default d_model
+    n_layers: ${n_layers}
+    d_model: ${d_model}
+
+    # Sub-sampling params
+    subsampling: striding # vggnet, striding, stacking or stacking_norm, dw_striding # dw_striding in fast conformer
+    subsampling_factor: 4 # must be power of 2 for striding and vggnet
+    subsampling_conv_channels: -1 # -1 sets it to d_model = 256, -1 in fast
+    causal_downsampling: false
+
+    # Feed forward module's params
+    ff_expansion_factor: 4
+
+    # Multi-headed Attention Module's params
+    self_attention_model: rel_pos # rel_pos or abs_pos
+    n_heads: ${n_heads} # may need to be lower for smaller d_models
+    # [left, right] specifies the number of steps to be seen from left and right of each step in self-attention
+    att_context_size: [-1, -1] # -1 means unlimited context
+    att_context_style: regular # regular or chunked_limited
+    xscaling: true # scales up the input embeddings by sqrt(d_model)
+    untie_biases: true # unties the biases of the TransformerXL layers
+    pos_emb_max_len: 5000
+
+    # Convolution module's params
+    conv_kernel_size: ${conv_kernel_size}
+    conv_norm_type: "batch_norm" # batch_norm or layer_norm or groupnormN (N specifies the number of groups)
+    # conv_context_size can be"causal" or a list of two integers while conv_context_size[0]+conv_context_size[1]+1==conv_kernel_size
+    # null means [(kernel_size-1)//2, (kernel_size-1)//2], and 'causal' means [(kernel_size-1), 0]
+    conv_context_size: null
+
+    ### regularization
+    dropout: 0.1 # The dropout used in most of the Conformer Modules
+    dropout_pre_encoder: 0.1 # The dropout used before the encoder
+    dropout_emb: 0.0 # The dropout used for embeddings
+    dropout_att: 0.1 # The dropout for multi-headed attention modules
+
+    # set to non-zero to enable stochastic depth
+    stochastic_depth_drop_prob: 0.0
+    stochastic_depth_mode: linear # linear or uniform
+    stochastic_depth_start_layer: 1
+
+  decoder:
+    _target_: nemo.collections.asr.modules.ConvASRDecoder
+    feat_in: null
+    num_classes: -1
+    vocabulary: []
+
+  # config for InterCTC loss: https://arxiv.org/abs/2102.03216
+  # specify loss weights and which layers to use for InterCTC
+  # e.g., to reproduce the paper results, set loss_weights: [0.3]
+  # and apply_at_layers: [8] (assuming 18 layers). Note that final
+  # layer loss coefficient is automatically adjusted (to 0.7 in above example)
+  interctc:
+    loss_weights: []
+    apply_at_layers: []
+
+  optim:
+    name: adamw
+    # lr: 5.0 #?
+    lr: ${lr}
+    # optimizer arguments
+    betas: [0.9, 0.98]
+    # less necessity for weight_decay as we already have large augmentations with SpecAug
+    # you may need weight_decay for large models, stable AMP training, small datasets, or when lower augmentations are used
+    # weight decay of 0.0 with lr of 2.0 also works fine
+    # weight_decay: 0.0 #?
+    weight_decay: ${weight_decay}
+
+    # scheduler setup
+    sched:
+      name: NoamAnnealing
+      d_model: ${model.encoder.d_model}
+      # scheduler config override
+      # warmup_steps: 10000
+      warmup_steps: ${warmup_steps}
+      warmup_ratio: null
+      min_lr: 1e-6
+
+# ==============================================================================
+# Dont use
+
+trainer:
+  devices: -1 # number of GPUs, -1 would use all available GPUs
+  num_nodes: 1
+  max_epochs: 1000
+  max_steps: -1 # computed at runtime if not set
+  val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations
+  accelerator: auto
+  strategy: ddp
+  accumulate_grad_batches: 1
+  gradient_clip_val: 0.0
+  precision: 32 # 16, 32, or bf16
+  log_every_n_steps: 10 # Interval of logging.
+  enable_progress_bar: True
+  num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it
+  check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs
+  sync_batchnorm: true
+  enable_checkpointing: False # Provided by exp_manager
+  logger: false # Provided by exp_manager
+  benchmark: false # needs to be false for models with variable-length speech input as it slows down training
+
+exp_manager:
+  exp_dir: null
+  name: ${name}
+  create_tensorboard_logger: true
+  create_checkpoint_callback: true
+  checkpoint_callback_params:
+    # in case of multiple validation sets, first one is used
+    monitor: "val_wer"
+    mode: "min"
+    save_top_k: 5
+    always_save_nemo: True # saves the checkpoints as nemo files instead of PTL checkpoints
+
+  # you need to set these two to True to continue the training
+  resume_if_exists: false
+  resume_ignore_no_checkpoint: false
+
+  # You may use this section to create a W&B logger
+  create_wandb_logger: false
+  wandb_logger_kwargs:
+    name: null
+    project: null

test_conformer_10-15_07-53/code-folder/train.py

@@ -0,0 +1,289 @@
+import os
+import subprocess
+import sys
+
+from sklearn.model_selection import train_test_split
+from torch import rand
+
+# * Have first: V0, utils -> this
+if True:
+    sys.path.append(os.getcwd())
+    if True:
+        from v0_import.import_scr import *
+    if True:
+        from v2_run.Conformer_nemo.utils.utils import *
+    run_import_src = True
+
+
+# ==============================================================================
+# Ultil functions
+# Dont need to edit, please..
+
+def set_all_seeds(seed=42):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+
+
+def get_config_nemo():
+    # config_path = "Fast_conformer_nemo/configs/fast_conformer.yaml"
+    # if len(sys.argv) >= 2:
+    #     config_path = sys.argv[1]
+    print(config_path, flush=True)
+    with open(config_path, "r", encoding="utf-8") as f:
+        yaml_text = f.read().strip()
+
+    _config = OmegaConf.create(yaml_text)
+    config_dict = OmegaConf.to_container(_config, resolve=True)
+    print(f"\n{config_dict['model']}\n{config_dict}", flush=True)
+    return config_dict
+
+
+def get_train_dev_test(path):  # Train is full, dev is 300, test is full
+    data = []
+    print(path, flush=True)
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            if line.strip():
+                dic = json.loads(line)
+                data.append(dic)
+    return data
+
+
+def get_info(train_path):
+    data = []
+    max_dur = 0.0
+    words = []
+    with open(train_path, "r", encoding="utf-8") as f:  # Same code with above
+        for line in f:
+            if line.strip():
+                dic = json.loads(line)
+                data.append(dic)
+                max_dur = max(max_dur, dic["duration"])
+                words.extend(str(dic["text"]).split())
+
+    words = list(set(sorted(words)))
+    return data, max_dur, len(words)
+
+
+# ==============================================================================
+# Set global vars and data
+if run_import_src:
+    set_all_seeds(42)  # Set random seed for reproducibility
+    config_dict = get_config_nemo()  # Get from path
+    data, max_dur, len_vocab = get_info(config_dict['train_path'])
+    sample_rate = 16000
+    dev_data, test_data = get_train_dev_test(
+        config_dict['dev_path']), get_train_dev_test(config_dict['test_path'])
+    full_val_data = get_train_dev_test(config_dict['full_val_path'])
+
+    log.info("Overall checking:")
+    log.info(f"Number of samples in manifest: {len(data)}")
+    log.info(
+        f"- Max duration in manifest: {max_dur:.2f} seconds, sample rate: {sample_rate}")
+    log.info(f"- Vocab in manifest: {len_vocab} units")
+    log.info(f"Number of dev is {len(dev_data)}, test is {len(test_data)}")
+
+
+# ==============================================================================
+# Create params, conf_model, mfpath and change st
+if run_import_src:
+    params = copy.deepcopy(config_dict)['model']
+    train_mfpath = os.path.join(res_exp_dir, "_train_mf.json")
+    dev_mfpath = os.path.join(res_exp_dir, "_dev_mf.json")
+
+    for ds in ['train_ds', 'validation_ds']:  # Change train and dev temporarily
+        if ds == 'train_ds':
+            params[ds]['manifest_filepath'] = train_mfpath
+            params[ds]['max_duration'] = round(max_dur + 0.1, 2)
+        else:
+            params[ds]['manifest_filepath'] = dev_mfpath
+    log.info(
+        f"Changed params['train_ds']['max_duration'] = {round(max_dur + 0.1, 2)} seconds \nand {train_mfpath}, {dev_mfpath} to write data!")
+
+    # Create an OmegaConf object from the dictionary
+    conf_model = OmegaConf.create(params)
+    write_txt_exp_dir("conf_model.txt", conf_model)
+    # Run pip freeze and capture output as string
+    pip_freeze_str = subprocess.check_output(
+        ["git", "log", "-1",
+            '--pretty=format:%h %s%nAuthor date: %ad%nCommit date: %cd', "--date=iso"],
+        text=True
+    ) + "\n"
+    pip_freeze_str += subprocess.check_output(
+        ["python", "--version"], text=True) + "\n"
+    pip_freeze_str += subprocess.check_output(
+        ["pip", "freeze"], text=True) + "\n"
+    write_txt_exp_dir("git_pip_env.txt", pip_freeze_str)
+
+    # Some for training
+    log.info(f"train_ds_batch_size: {config_dict['train_ds_batch_size']}")
+    log.info(
+        f"warmup_steps: {config_dict['warmup_steps']}, lr: {config_dict['lr']}, weight_decay: {config_dict['weight_decay']}")
+
+
+# ==============================================================================
+# Create subset from part and write it to global mfpath
+
+
+def create_train_dev(root_train_data: list, begin_idx, end_idx, num_train, num_dev, root_dev_data=None):
+    global train_mfpath, dev_mfpath
+
+    def get_min_max_duration(data):
+        mi, ma = 100.0, 0.0
+        for dic in data:
+            mi = min(mi, dic["duration"])
+            ma = max(ma, dic["duration"])
+        return mi, ma
+
+    def split_data(data: list, k):
+        if k >= len(data):
+            return data
+        _train_data, _test_data = train_test_split(
+            data, test_size=k, random_state=42
+        )
+        return _test_data
+
+    # # Sort by distribution
+    # data = list(sorted(root_train_data, key=lambda x: int(
+    #     os.path.basename(x["audio_filepath"]).split("_")[0])))
+
+    # Select part
+    data = root_train_data.copy()
+    begin_idx = max(0, begin_idx)
+    end_idx = min(end_idx, len(data))
+    data = data[begin_idx:end_idx]
+    log.info(f"- Duration of this part: [{get_min_max_duration(data)}]")
+
+    # Select random subset/set from part: train from data, and dev from train or is root_dev
+    from tqdm import tqdm
+    num_train = min(num_train, len(data))
+    train_data = random.sample(data, num_train)  # ? Use random.sample
+    if root_dev_data is None:
+        num_dev = min(num_dev, num_train)
+        dev_data = random.sample(train_data, num_dev)  # ? Use random.sample
+    else:
+        dev_data = list(root_dev_data).copy()
+
+    # Write subset data back to a new file (or overwrite)
+    dev_in_train = True if root_dev_data is None else False
+    log.info(
+        f"- Number of train is {len(train_data)}, dev is {len(dev_data)}, dev in train: {dev_in_train}")
+    log.info(f"\n{train_data[0]}\n{dev_data[0]}\n")
+
+    with open(train_mfpath, "w", encoding="utf-8") as fout:
+        for item in train_data:
+            fout.write(json.dumps(item, ensure_ascii=False) + "\n")
+
+    with open(dev_mfpath, "w", encoding="utf-8") as fout:
+        for item in dev_data:
+            fout.write(json.dumps(item, ensure_ascii=False) + "\n")
+
+
+# ==============================================================================
+# Demo Nemo model and demo first
+def init_nemo_model(data, dev_data, conf_model):
+    log.info(f"\n\nInit nemo model:")
+    create_train_dev(root_train_data=data, begin_idx=0, end_idx=len(data), num_train=len(
+        data), num_dev=-1, root_dev_data=dev_data)  # Demo create data, v
+    trainer = create_new_trainer(epochs=1000, min_stop=0.0)  # Demo trainer, v
+
+    # trainer.fit(nemo_model)
+    nemo_model = nemo_asr.models.EncDecCTCModelBPE(
+        cfg=conf_model, trainer=trainer)
+    summary = ModelSummary(nemo_model)
+    print(summary)
+    return nemo_model
+
+
+# ==============================================================================
+# Train model: Multi part training, need edit for Conformer
+
+
+def train_multi_turn(train_data, dev_data):
+    # Init nemo model
+    set_all_seeds(42)
+    global params, conf_model, res_exp_dir
+    nemo_model = init_nemo_model(
+        data=train_data, dev_data=dev_data, conf_model=conf_model)
+
+    # ! NOTE: Setting for multi part training
+    train_data = list(sorted(train_data, key=lambda x: x["duration"]))
+
+    cnt = len(train_data)
+    setting = {
+        "begin_idx": [0] * 5 + [0],  # ! Difference with root version
+        "end_idx": [3000, 6000, 9000, 12000, 15000, cnt],
+        "num_train": [300] * 5 + [cnt],
+        "num_dev": [10] * 5 + [-1],  # Must -1
+        "epochs": [200] * 5 + [1000],  # ! Not run enough epochs
+        "min_stop": [0.3] * 5 + [0.0]  # ! 0.3
+    }
+    num_part = len(setting["begin_idx"])
+    trainer = None
+
+    for i in range(num_part):
+        begin_idx = setting["begin_idx"][i]
+        end_idx = setting["end_idx"][i]
+        num_train = setting["num_train"][i]
+        num_dev = setting["num_dev"][i]
+        num_epochs = setting["epochs"][i]
+        min_stop = setting["min_stop"][i]
+        log.info(
+            f"\n\n Here {i}: {begin_idx} --> {end_idx} | {num_train}, {num_dev}, {cnt} | {num_epochs}, {min_stop}")
+
+        # Create train dev file
+        if num_dev <= 0:
+            create_train_dev(root_train_data=train_data, begin_idx=begin_idx, end_idx=end_idx,
+                             num_train=num_train, num_dev=num_dev, root_dev_data=dev_data)
+        else:
+            create_train_dev(root_train_data=train_data, begin_idx=begin_idx, end_idx=end_idx,
+                             num_train=num_train, num_dev=num_dev)
+
+        # Create trainer newly
+        trainer = create_new_trainer(
+            epochs=num_epochs, min_stop=min_stop)
+
+        nemo_model.setup_training_data(
+            train_data_config=params['train_ds'])  # Reload it
+        nemo_model.setup_validation_data(
+            val_data_config=params['validation_ds'])  # Reload it
+        trainer.fit(nemo_model)  # Fit
+
+        # Save it tmply
+        save_path = os.path.join(res_exp_dir, f"_nemo_model_part_{i}.nemo")
+        nemo_model.save_to(save_path)
+    return trainer, nemo_model
+
+
+# =====================================
+# Conformer, dont need to edit, please..
+
+
+def run_main_in_notebook():
+    global data, dev_data, res_exp_dir
+    trainer, nemo_model = train_multi_turn(data, dev_data)  # Train with data
+
+    # ==========================================================================
+    # Get paths from checkpoint callback
+    # # last in list of callbacks if added last
+    ckpt_callback = trainer.callbacks[-1]
+    best_paths = list(ckpt_callback.best_k_models.keys())
+
+    # Reload and save
+    nemo_model, avg_weights = reload_nemo_from_avg(
+        best_paths=best_paths, nemo_model=nemo_model)
+    avg_ckpt_path = f"{res_exp_dir}/model_avg.ckpt"
+    nemo_model_path = f"{res_exp_dir}/nemo_model_avg.nemo"
+    save_model_to_path(nemo_model, avg_weights, nemo_model_path, avg_ckpt_path)
+
+    # Gen, score and save for mfpath
+    nemo_inference_for_mfpath(nemo_model, config_dict['dev_path'])
+    nemo_inference_for_mfpath(nemo_model, config_dict['test_path'])
+    nemo_inference_for_mfpath(nemo_model, config_dict['full_val_path'])
+    push_exp_dir_to_hub(res_exp_dir)
+
+
+if __name__ == "__main__":
+    run_main_in_notebook()

test_conformer_10-15_07-53/code-folder/utils/utils.py

@@ -0,0 +1,300 @@
+import copy
+import csv
+import glob
+import json
+import logging
+import logging as log
+import os
+import random
+import re
+import shutil
+import string
+import sys
+import unicodedata
+
+import jiwer
+import lightning.pytorch as pl
+import nemo
+import nemo.collections.asr as nemo_asr
+import numpy as np
+import torch
+from datasets import load_dataset
+from jiwer import wer
+from lightning.pytorch.callbacks import Callback, EarlyStopping, ModelCheckpoint
+from lightning.pytorch.utilities.model_summary import ModelSummary
+from omegaconf import OmegaConf
+from scipy.io import wavfile
+
+# * Have first: V0 -> this
+from v0_import.import_scr import push_file_to_hub
+
+
+class LossLogger(Callback):
+    def __init__(self, exp_dir):
+        super().__init__()
+        self.train_losses = []
+        self.val_losses = []
+        self.train_wer = []
+        self.val_wer = []
+        self.lr_list = []  # ? lr plot
+        self.step_list = []  # ? step plot
+        self.num_last = 100  # ? epoch unit
+        self.num_plot = 100  # ? epoch
+        self.allow_show_plot = False  # ? Allow show plot in notebook
+        self.exp_dir = exp_dir
+
+    def on_train_epoch_end(self, trainer, pl_module):
+        train_loss = trainer.callback_metrics.get('train_loss')
+        epoch_idx = trainer.current_epoch
+        lr = trainer.optimizers[0].param_groups[0]['lr']  # Print lr
+        optimize_step = trainer.global_step  # <-- this is what you want
+        log.info(f"Epoch {epoch_idx} ended." + "=" * 100)
+        if train_loss is not None:
+            self.train_losses.append(train_loss.item())
+            self.lr_list.append(lr)  # Add lr
+            self.step_list.append(optimize_step)  # Add step
+            log.info(
+                f"Train Loss: {train_loss.item()}, lr: {lr}, step: {optimize_step}")
+
+        if epoch_idx != 0 and epoch_idx % self.num_plot == 0:
+            self._plot_train()
+
+    def on_validation_epoch_end(self, trainer, pl_module):
+        val_loss = trainer.callback_metrics.get('val_loss')
+        val_wer = trainer.callback_metrics.get('val_wer')
+        if val_loss is not None:
+            self.val_losses.append(val_loss.item())
+            log.info(f"Validation Loss: {val_loss.item()}")
+        if val_wer is not None:
+            self.val_wer.append(val_wer.item())
+            log.info(f"Validation WER: {val_wer.item()}")
+
+    def _plot_train(self):
+        import matplotlib.pyplot as plt
+        plt.figure(figsize=(20, 16))  # Bigger figure
+        num = self.num_last
+        # ===== Loss Plot =====
+        plt.subplot(2, 2, 1)
+        plt.plot(self.train_losses[-num:], label='Training Loss', linewidth=1)
+        plt.plot(self.val_losses[-num:], label='Validation Loss', linewidth=1)
+        plt.xlabel('Epoch')
+        plt.ylabel('Loss')
+        plt.legend()
+        plt.title('Training and Validation Loss')
+        plt.grid(True, linestyle='--', alpha=0.6)
+        # ===== WER Plot =====
+        plt.subplot(2, 2, 2)
+        plt.plot(self.train_wer[-num:], label='Training WER', linewidth=1)
+        plt.plot(self.val_wer[-num:], label='Validation WER', linewidth=1)
+        plt.xlabel('Epoch')
+        plt.ylabel('WER')
+        plt.legend()
+        plt.title('Training and Validation WER')
+        plt.grid(True, linestyle='--', alpha=0.6)
+        # ===== Learning Rate Plot =====
+        plt.subplot(2, 2, 3)
+        plt.plot(self.lr_list[-num:], label='Learning rate', linewidth=1)
+        plt.xlabel('Epoch')
+        plt.ylabel('LR')
+        plt.legend()
+        plt.title('Learning Rate Schedule')
+        plt.grid(True, linestyle='--', alpha=0.6)
+        # ===== Optimize step Plot =====
+        plt.subplot(2, 2, 4)
+        plt.plot(self.step_list[-num:], label='Optimize step', linewidth=1)
+        plt.xlabel('Epoch')
+        plt.ylabel('Step')
+        plt.legend()
+        plt.title('Step Optimization')
+        plt.grid(True, linestyle='--', alpha=0.6)
+        plt.tight_layout()
+        # allow_show_plot = True  # Allow show plot in notebook
+        if self.allow_show_plot:
+            plt.show()
+        else:
+            plot_png = os.path.join(
+                self.exp_dir, f"training_process_{len(self.val_wer)}.png")
+            plt.savefig(plot_png)
+            push_file_to_hub(plot_png)
+
+    def on_train_end(self, trainer, pl_module):
+        self.num_last = len(self.val_wer)
+        self._plot_train()
+
+
+config_path = "v2_run/Conformer_nemo/configs/conformer.yaml"  # ! NOTE: Setting
+res_exp_dir = "test_conformer"  # ? NOTE: Setting
+os.makedirs(res_exp_dir, exist_ok=True)
+src_folder = "v2_run/Conformer_nemo"  # ?
+dst_folder = os.path.join(res_exp_dir, "code-folder")
+shutil.copytree(src_folder, dst_folder, dirs_exist_ok=True)
+log.info(f"Copied code to {dst_folder}")
+
+
+def write_txt_exp_dir(name, var):
+    path = os.path.join(res_exp_dir, name)
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(str(var))
+        f.close()
+
+
+# ==============================================================================
+
+
+def create_time_callbacks(num_keep, min_stop, max_hour):
+    # num_keep = 500
+    early_stop_callback = EarlyStopping(
+        monitor="val_wer",                  # Metric to monitor
+        mode="min",                         # Lower is better
+        stopping_threshold=min_stop,        # Stop if val_wer < 0.x
+        patience=num_keep,                  # Stop immediately when not reduce
+        verbose=True
+    )
+    # Keep top 5 checkpoints based on val_wer
+    num_avg = 5
+    save_last = False
+    checkpoint_callback = ModelCheckpoint(
+        dirpath=f"{res_exp_dir}/ckpts",     # Dir of ckpts
+        filename="epoch{epoch}-{val_wer:.4f}",
+        monitor="val_wer",  # ! Use val_cer metric
+        mode="min",
+        save_top_k=num_avg,                 # Only keep 5 best
+        save_last=save_last,                # Also save last epoch: False
+    )
+    # max_time_training = "00:09:00:00"
+    max_time_training = f"00:{max_hour}:02:00"
+    callback_list = [LossLogger(res_exp_dir),
+                     early_stop_callback, checkpoint_callback]  # Difference with root version
+    return max_time_training, callback_list
+
+
+def create_new_trainer(epochs, min_stop, max_hour="09"):
+    # NOTE: Setting
+    max_hour = "00"  # ! Must edit when run: Conformer
+    log.info(f"Hour to train is {max_hour}")
+    setting = {
+        'num_keep': 500,
+        'precision': 'bf16',  # ! Use AMP: Difference with root version
+        'accumulate_grad_batches': 1,
+        'max_hour': max_hour,
+        'enable_progress_bar': False,   # Off bar training to shorter log
+    }
+    log.info(f"Precision to train is {setting['precision']}")
+    log.info(
+        f"Grad batch size to train is {16} x {setting['accumulate_grad_batches']}")  # ! Bsize
+
+    # Create callbacks
+    max_time_training, callback_list = create_time_callbacks(
+        num_keep=setting['num_keep'], min_stop=min_stop, max_hour=max_hour)
+    # Training args
+    trainer_dict = {
+        # Hardware
+        'precision': setting['precision'],  # Trade-off
+        'devices': 1,
+        'num_nodes': 1,
+        'accelerator': 'gpu',
+        'strategy': 'auto',  # Must: no multi gpu
+        # Training
+        'max_epochs': epochs,
+        'accumulate_grad_batches': setting['accumulate_grad_batches'],
+        'gradient_clip_val': 0.0,
+        # Prediction monitor
+        'log_every_n_steps': 100,  # Logging in a epoch train
+        'val_check_interval': 1.0,  # Compute wer after 1.0 epoch
+        # No-related
+        'enable_progress_bar': setting['enable_progress_bar'],
+        'num_sanity_val_steps': 0,
+        'check_val_every_n_epoch': 1,
+        # If True, enables cudnn benchmarking for faster training.
+        'sync_batchnorm': True,
+        'benchmark': False,
+        # Saving and callback: New setting for callbacks
+        'enable_checkpointing': True,
+        'max_time': max_time_training,
+        'callbacks': callback_list,
+    }
+    write_txt_exp_dir("args_trainer.txt", trainer_dict)
+    trainer = pl.Trainer(**trainer_dict)
+    return trainer
+
+
+# ==============================================================================
+# Dont need to edit, please..
+
+
+def reload_nemo_from_avg(best_paths, nemo_model):
+    w_only = False  # NOTE: Use w_only = False because it error
+    load_strict = False
+
+    def average_checkpoints(paths):
+        avg_state_dict = None
+        for path in paths:
+            ckpt = torch.load(path, map_location="cpu",
+                              weights_only=w_only)["state_dict"]
+            if avg_state_dict is None:
+                avg_state_dict = {k: v.clone() for k, v in ckpt.items()}
+            else:
+                for k in avg_state_dict:
+                    # if it's int/bool, leave as-is
+                    if torch.is_floating_point(avg_state_dict[k]):
+                        avg_state_dict[k] += ckpt[k]
+        for k in avg_state_dict:
+            if torch.is_floating_point(avg_state_dict[k]):
+                avg_state_dict[k] /= len(paths)
+        return avg_state_dict
+
+    # Average
+    log.info(f"\n\nBest paths for AVG(model): {best_paths}")
+    avg_weights = average_checkpoints(best_paths)
+    # Assign averaged weights to NeMo model
+    nemo_model = nemo_model.to("cuda" if torch.cuda.is_available() else "cpu")
+    nemo_model.load_state_dict(avg_weights, strict=load_strict)
+    return nemo_model, avg_weights
+
+
+def save_model_to_path(nemo_model, avg_weights, nemo_model_path, avg_ckpt_path):
+    torch.save({"state_dict": avg_weights}, avg_ckpt_path)
+    nemo_model.save_to(nemo_model_path)
+    log.info(f"\n\nSaved avg weights (.ckpt) at {avg_ckpt_path}")
+    log.info(f"Saved averaged NeMo model at {nemo_model_path}")
+
+
+def nemo_inference_for_mfpath(nemo_model, mfpath):
+    def save_gen_list(text_list, gt_list):
+        random_name = ''.join(random.choices(
+            string.ascii_lowercase + string.digits, k=8))
+        file_path = f"{random_name}.csv"
+        # Save rd name
+        file_path = os.path.join(res_exp_dir, file_path)
+        log.info(f"Saved gen at {file_path}")
+        # Write it as .csv
+        with open(file_path, mode="w", newline="", encoding="utf-8") as f:
+            writer = csv.writer(f)
+            writer.writerow(["Gen", "GT"])  # header
+            for first, second in zip(text_list, gt_list):
+                writer.writerow([first, second])
+
+    with open(mfpath, "r", encoding="utf-8") as fin:
+        data = [json.loads(line) for line in fin]
+    log.info(f"\n\nLoaded {len(data)} entries from {mfpath}")
+
+    references = []
+    predictions = []
+    from tqdm import tqdm
+    for entry in data:  # Limit data if need
+        ref = entry['text']
+        audio_path = entry['audio_filepath']
+        with torch.no_grad():
+            pred = nemo_model.transcribe(audio_path, verbose=False)[0].text
+        # if use_norm:
+        #     pred = normalize_text_vietnamese(pred)
+        references.append(ref)
+        predictions.append(pred)
+
+    # Computer wer
+    wer_score = wer(references, predictions)
+    log.info(f"WER: {wer_score}")
+
+    # Save pred
+    save_gen_list(text_list=predictions, gt_list=references)
+    return wer_score

test_conformer_10-15_07-53/conf_model.txt

@@ -0,0 +1 @@
+{'sample_rate': 16000, 'log_prediction': False, 'ctc_reduction': 'mean_batch', 'skip_nan_grad': False, 'wer': {'use_cer': True}, 'use_cer': True, 'train_ds': {'manifest_filepath': 'test_conformer/_train_mf.json', 'sample_rate': 16000, 'batch_size': 16, 'shuffle': True, 'num_workers': 8, 'pin_memory': True, 'max_duration': 30.44, 'min_duration': 0.1, 'is_tarred': False, 'tarred_audio_filepaths': None, 'shuffle_n': 2048, 'bucketing_strategy': 'synced_randomized', 'bucketing_batch_size': None}, 'validation_ds': {'manifest_filepath': 'test_conformer/_dev_mf.json', 'sample_rate': 16000, 'batch_size': 16, 'shuffle': False, 'use_start_end_token': False, 'num_workers': 8, 'pin_memory': True}, 'test_ds': {'manifest_filepath': 'v2_prepdata/out/test_300.json', 'sample_rate': 16000, 'batch_size': 16, 'shuffle': False, 'use_start_end_token': False, 'num_workers': 8, 'pin_memory': True}, 'tokenizer': {'dir': 'v2_prepdata/out/', 'type': 'wpe'}, 'preprocessor': {'_target_': 'nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor', 'sample_rate': 16000, 'normalize': 'per_feature', 'window_size': 0.025, 'window_stride': 0.01, 'window': 'hann', 'features': 80, 'n_fft': 512, 'log': True, 'frame_splicing': 1, 'dither': 1e-05, 'pad_to': 0, 'pad_value': 0.0}, 'spec_augment': {'_target_': 'nemo.collections.asr.modules.SpectrogramAugmentation', 'freq_masks': 2, 'time_masks': 5, 'freq_width': 27, 'time_width': 0.05}, 'encoder': {'_target_': 'nemo.collections.asr.modules.ConformerEncoder', 'feat_in': 80, 'feat_out': -1, 'n_layers': 16, 'd_model': 176, 'subsampling': 'striding', 'subsampling_factor': 4, 'subsampling_conv_channels': -1, 'causal_downsampling': False, 'ff_expansion_factor': 4, 'self_attention_model': 'rel_pos', 'n_heads': 4, 'att_context_size': [-1, -1], 'att_context_style': 'regular', 'xscaling': True, 'untie_biases': True, 'pos_emb_max_len': 5000, 'conv_kernel_size': 31, 'conv_norm_type': 'batch_norm', 'conv_context_size': None, 'dropout': 0.1, 'dropout_pre_encoder': 0.1, 'dropout_emb': 0.0, 'dropout_att': 0.1, 'stochastic_depth_drop_prob': 0.0, 'stochastic_depth_mode': 'linear', 'stochastic_depth_start_layer': 1}, 'decoder': {'_target_': 'nemo.collections.asr.modules.ConvASRDecoder', 'feat_in': None, 'num_classes': -1, 'vocabulary': []}, 'interctc': {'loss_weights': [], 'apply_at_layers': []}, 'optim': {'name': 'adamw', 'lr': 5.0, 'betas': [0.9, 0.98], 'weight_decay': 0.0, 'sched': {'name': 'NoamAnnealing', 'd_model': 176, 'warmup_steps': 10000, 'warmup_ratio': None, 'min_lr': 1e-06}}}

test_conformer_10-15_07-53/git_pip_env.txt

@@ -0,0 +1,878 @@
+eb31208 try conformer
+Author date: 2025-10-15 14:38:28 +0700
+Commit date: 2025-10-15 14:38:28 +0700
+Python 3.11.11
+
+absl-py==1.4.0
+accelerate==1.5.2
+aiofiles==22.1.0
+aiohappyeyeballs==2.6.1
+aiohttp==3.11.18
+aiosignal==1.3.2
+aiosqlite==0.21.0
+alabaster==1.0.0
+albucore==0.0.23
+albumentations==2.0.5
+ale-py==0.10.2
+alembic==1.15.2
+altair==5.5.0
+annotated-types==0.7.0
+annoy==1.17.3
+ansicolors==1.1.8
+antlr4-python3-runtime==4.9.3
+anyio==4.9.0
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+args==0.1.0
+array_record==0.7.1
+arrow==1.3.0
+arviz==0.21.0
+astropy==7.0.1
+astropy-iers-data==0.2025.3.31.0.36.18
+asttokens==3.0.0
+astunparse==1.6.3
+atpublic==5.1
+attrs==25.3.0
+audioread==3.0.1
+autograd==1.7.0
+babel==2.17.0
+backcall==0.2.0
+bayesian-optimization==2.0.3
+beartype==0.20.2
+beautifulsoup4==4.13.3
+betterproto==2.0.0b6
+bigframes==1.42.0
+bigquery-magics==0.9.0
+bitsandbytes==0.46.0
+bleach==6.2.0
+blinker==1.9.0
+blis==1.2.1
+blobfile==3.0.0
+blosc2==3.2.1
+bokeh==3.6.3
+Boruta==0.4.3
+boto3==1.38.11
+botocore==1.38.11
+Bottleneck==1.4.2
+-e git+https://github.com/SohierDane/BigQuery_Helper@8615a7f6c1663e7f2d48aa2b32c2dbcb600a440f#egg=bq_helper
+bqplot==0.12.44
+braceexpand==0.1.7
+branca==0.8.1
+CacheControl==0.14.2
+cachetools==5.5.2
+Cartopy==0.24.1
+catalogue==2.0.10
+catboost==1.2.8
+category_encoders==2.7.0
+certifi==2025.4.26
+cesium==0.12.4
+cffi==1.17.1
+chardet==5.2.0
+charset-normalizer==3.4.2
+Chessnut==0.4.1
+chex==0.1.89
+clarabel==0.10.0
+click==8.1.8
+click-plugins==1.1.1
+cligj==0.7.2
+clint==0.5.1
+cloudpathlib==0.21.0
+cloudpickle==3.1.1
+cmake==3.31.6
+cmdstanpy==1.2.5
+colorama==0.4.6
+colorcet==3.1.0
+colorlog==6.9.0
+colorlover==0.3.0
+colour==0.1.5
+comm==0.2.2
+community==1.0.0b1
+confection==0.1.5
+cons==0.4.6
+contourpy==1.3.1
+coverage==7.8.0
+cramjam==2.9.1
+cryptography==44.0.3
+cuda-bindings==12.9.0
+cuda-python==12.9.0
+cudf-cu12==25.2.2
+cudf-polars-cu12==25.2.2
+cufflinks==0.17.3
+cuml-cu12==25.2.1
+cupy-cuda12x==13.4.1
+cuvs-cu12==25.2.1
+cvxopt==1.3.2
+cvxpy==1.6.4
+cycler==0.12.1
+cyipopt==1.5.0
+cymem==2.0.11
+Cython==3.0.12
+cytoolz==1.0.1
+daal==2025.5.0
+dacite==1.9.2
+dask==2024.12.1
+dask-cuda==25.2.0
+dask-cudf-cu12==25.2.2
+dask-expr==1.1.21
+dataclasses-json==0.6.7
+datascience==0.17.6
+datasets==3.6.0
+db-dtypes==1.4.2
+dbus-python==1.2.18
+deap==1.4.3
+debugpy==1.8.0
+decorator==4.4.2
+deepdiff==8.4.2
+defusedxml==0.7.1
+Deprecated==1.2.18
+diffusers==0.32.2
+dill==0.3.8
+dipy==1.11.0
+distributed==2024.12.1
+distributed-ucxx-cu12==0.42.0
+distro==1.9.0
+dlib==19.24.6
+dm-tree==0.1.9
+dnspython==2.7.0
+docker==7.1.0
+docker-pycreds==0.4.0
+docopt==0.6.2
+docstring-to-markdown==0.17
+docstring_parser==0.16
+docutils==0.21.2
+dopamine_rl==4.1.2
+duckdb==1.2.1
+earthengine-api==1.5.9
+easydict==1.13
+easyocr==1.7.2
+editdistance==0.8.1
+eerepr==0.1.1
+einops==0.8.1
+eli5==0.13.0
+email_validator==2.2.0
+emoji==2.14.1
+en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl#sha256=1932429db727d4bff3deed6b34cfc05df17794f4a52eeb26cf8928f7c1a0fb85
+entrypoints==0.4
+et_xmlfile==2.0.0
+etils==1.12.2
+etuples==0.3.9
+execnb==0.1.14
+Farama-Notifications==0.0.4
+fastai==2.7.19
+fastcore==1.7.29
+fastdownload==0.0.7
+fastjsonschema==2.21.1
+fastprogress==1.0.3
+fastrlock==0.8.3
+fasttext==0.9.3
+featuretools==1.31.0
+fiddle==0.3.0
+filelock==3.18.0
+fiona==1.10.1
+firebase-admin==6.7.0
+Flask==3.1.0
+flatbuffers==25.2.10
+flax==0.10.5
+folium==0.19.5
+fonttools==4.57.0
+fqdn==1.5.1
+frozendict==2.4.6
+frozenlist==1.6.0
+fsspec==2024.12.0
+funcy==2.0
+fury==0.12.0
+future==1.0.0
+fuzzywuzzy==0.18.0
+gast==0.6.0
+gatspy==0.3
+gcsfs==2025.3.2
+GDAL==3.6.4
+gdown==5.2.0
+geemap==0.35.3
+gensim==4.3.3
+geocoder==1.38.1
+geographiclib==2.0
+geojson==3.2.0
+geopandas==0.14.4
+geopy==2.4.1
+ghapi==1.0.6
+gin-config==0.5.0
+gitdb==4.0.12
+GitPython==3.1.44
+glob2==0.7
+google==2.0.3
+google-ai-generativelanguage==0.6.15
+google-api-core==1.34.1
+google-api-python-client==2.164.0
+google-auth==2.40.1
+google-auth-httplib2==0.2.0
+google-auth-oauthlib==1.2.1
+google-cloud-aiplatform==1.87.0
+google-cloud-automl==1.0.1
+google-cloud-bigquery==3.25.0
+google-cloud-bigquery-connection==1.18.2
+google-cloud-bigtable==2.30.0
+google-cloud-core==2.4.3
+google-cloud-dataproc==5.18.1
+google-cloud-datastore==2.20.2
+google-cloud-firestore==2.20.1
+google-cloud-functions==1.20.2
+google-cloud-iam==2.18.3
+google-cloud-language==2.17.1
+google-cloud-pubsub==2.29.0
+google-cloud-resource-manager==1.14.2
+google-cloud-spanner==3.53.0
+google-cloud-storage==2.19.0
+google-cloud-translate==3.12.1
+google-cloud-videointelligence==2.16.1
+google-cloud-vision==3.10.1
+google-colab @ file:///colabtools/dist/google_colab-1.0.0.tar.gz
+google-crc32c==1.7.1
+google-genai==1.9.0
+google-generativeai==0.8.4
+google-pasta==0.2.0
+google-resumable-media==2.7.2
+google-spark-connect==0.5.2
+googleapis-common-protos==1.70.0
+googledrivedownloader==1.1.0
+gpxpy==1.6.2
+graphviz==0.20.3
+greenlet==3.1.1
+grpc-google-iam-v1==0.14.2
+grpc-interceptor==0.15.4
+grpcio==1.72.0rc1
+grpcio-status==1.49.0rc1
+grpclib==0.4.8
+gspread==6.2.0
+gspread-dataframe==4.0.0
+gym==0.25.2
+gym-notices==0.0.8
+gymnasium==0.29.0
+h11==0.14.0
+h2==4.2.0
+h2o==3.46.0.7
+h5netcdf==1.6.1
+h5py==3.13.0
+haversine==2.9.0
+hdbscan==0.8.40
+hep_ml==0.7.3
+hf-xet==1.1.0
+hf_transfer==0.1.9
+highspy==1.9.0
+holidays==0.69
+holoviews==1.20.2
+hpack==4.1.0
+html5lib==1.1
+htmlmin==0.1.12
+httpcore==1.0.7
+httpimport==1.4.1
+httplib2==0.22.0
+httpx==0.28.1
+huggingface-hub==0.31.1
+humanize==4.12.2
+hydra-core==1.3.2
+hyperframe==6.1.0
+hyperopt==0.2.7
+ibis-framework==9.5.0
+id==1.5.0
+idna==3.10
+igraph==0.11.8
+ImageHash==4.3.1
+imageio==2.37.0
+imageio-ffmpeg==0.6.0
+imagesize==1.4.1
+imbalanced-learn==0.13.0
+immutabledict==4.2.1
+importlib_metadata==8.7.0
+importlib_resources==6.5.2
+imutils==0.5.4
+in-toto-attestation==0.9.3
+indic_numtowords==1.1.0
+inflect==7.5.0
+iniconfig==2.1.0
+intel-cmplr-lib-rt==2024.2.0
+intel-cmplr-lib-ur==2024.2.0
+intel-openmp==2024.2.0
+intervaltree==3.1.0
+ipyevents==2.0.2
+ipyfilechooser==0.6.0
+ipykernel==6.17.1
+ipyleaflet==0.19.2
+ipympl==0.9.7
+ipyparallel==8.8.0
+ipython==7.34.0
+ipython-genutils==0.2.0
+ipython-sql==0.5.0
+ipytree==0.2.2
+ipywidgets==8.1.5
+isoduration==20.11.0
+isoweek==1.3.3
+itsdangerous==2.2.0
+Janome==0.5.0
+jax==0.5.2
+jax-cuda12-pjrt==0.5.1
+jax-cuda12-plugin==0.5.1
+jaxlib==0.5.1
+jedi==0.19.2
+jeepney==0.7.1
+jellyfish==1.1.0
+jieba==0.42.1
+Jinja2==3.1.6
+jiter==0.9.0
+jiwer==3.1.0
+jmespath==1.0.1
+joblib==1.5.0
+json5==0.12.0
+jsonpatch==1.33
+jsonpickle==4.0.5
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+jupyter-console==6.1.0
+jupyter-events==0.12.0
+jupyter-leaflet==0.19.2
+jupyter-lsp==1.5.1
+jupyter-ydoc==0.2.5
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+jupyter_server==2.12.5
+jupyter_server_fileid==0.9.3
+jupyter_server_terminals==0.5.3
+jupyter_server_ydoc==0.8.0
+jupyterlab==3.6.8
+jupyterlab-lsp==3.10.2
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.27.3
+jupyterlab_widgets==3.0.13
+kaggle==1.7.4.2
+kaggle-environments==1.16.11
+kagglehub==0.3.12
+kaldi-python-io==1.2.2
+keras==3.8.0
+keras-core==0.1.7
+keras-cv==0.9.0
+keras-hub==0.18.1
+keras-nlp==0.18.1
+keras-tuner==1.4.7
+keyring==23.5.0
+kiwisolver==1.4.8
+kornia==0.8.1
+kornia_rs==0.1.9
+kt-legacy==1.0.5
+langchain==0.3.22
+langchain-core==0.3.50
+langchain-text-splitters==0.3.7
+langcodes==3.5.0
+langid==1.1.6
+langsmith==0.3.23
+language_data==1.3.0
+launchpadlib==1.10.16
+lazr.restfulclient==0.14.4
+lazr.uri==1.0.6
+lazy_loader==0.4
+learntools @ git+https://github.com/Kaggle/learntools@9188cafa2795c2cb720981631280853a7e55649c
+Levenshtein==0.27.1
+lhotse==1.31.1
+libclang==18.1.1
+libcst==1.8.5
+libcudf-cu12==25.2.2
+libcugraph-cu12==25.2.0
+libcuml-cu12==25.2.1
+libcuvs-cu12==25.2.1
+libkvikio-cu12==25.2.1
+libpysal==4.9.2
+libraft-cu12==25.2.0
+librosa==0.11.0
+libucx-cu12==1.18.1
+libucxx-cu12==0.42.0
+lightgbm @ file:///tmp/lightgbm/lightgbm-4.6.0-py3-none-linux_x86_64.whl
+lightning==2.4.0
+lightning-utilities==0.14.3
+lilcom==1.8.1
+lime==0.2.0.1
+line_profiler==4.2.0
+linkify-it-py==2.0.3
+llvmlite==0.43.0
+lml==0.2.0
+locket==1.0.0
+logical-unification==0.4.6
+loguru==0.7.3
+lxml==5.3.1
+Mako==1.3.10
+mamba==0.11.3
+marisa-trie==1.2.1
+Markdown==3.7
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+marshmallow==3.26.1
+matplotlib==3.7.2
+matplotlib-inline==0.1.7
+matplotlib-venn==1.1.2
+mdit-py-plugins==0.4.2
+mdurl==0.1.2
+mediapy==1.1.6
+miniKanren==1.0.3
+missingno==0.5.2
+mistune==0.8.4
+mizani==0.13.2
+mkl==2025.1.0
+mkl-fft==1.3.8
+mkl-random==1.2.4
+mkl-service==2.4.1
+mkl-umath==0.1.1
+ml-dtypes==0.4.1
+mlcrate==0.2.0
+mlxtend==0.23.4
+mne==1.9.0
+model-signing==1.0.1
+more-itertools==10.6.0
+moviepy==1.0.3
+mpld3==0.5.10
+mpmath==1.3.0
+msgpack==1.1.0
+multidict==6.4.3
+multimethod==1.12
+multipledispatch==1.0.0
+multiprocess==0.70.16
+multitasking==0.0.11
+murmurhash==1.0.12
+music21==9.3.0
+mypy_extensions==1.1.0
+namex==0.0.8
+narwhals==1.33.0
+natsort==8.4.0
+nbclassic==1.2.0
+nbclient==0.5.13
+nbconvert==6.4.5
+nbdev==2.3.36
+nbformat==5.10.4
+ndindex==1.9.2
+nemo-toolkit==2.5.0
+nest-asyncio==1.6.0
+networkx==3.4.2
+nibabel==5.3.2
+nilearn==0.10.4
+ninja==1.11.1.4
+nltk==3.9.1
+notebook==6.5.4
+notebook_shim==0.2.4
+num2words==0.5.14
+numba==0.60.0
+numba-cuda==0.2.0
+numexpr==2.10.2
+numpy==1.26.4
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cuda-nvcc-cu12==12.5.82
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-runtime-cu12==12.4.127
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.2.1.3
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cusparselt-cu12==0.6.2
+nvidia-ml-py==12.575.51
+nvidia-nccl-cu12==2.21.5
+nvidia-nvcomp-cu12==4.2.0.11
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.4.127
+nvtx==0.2.11
+nx-cugraph-cu12 @ https://pypi.nvidia.com/nx-cugraph-cu12/nx_cugraph_cu12-25.2.0-py3-none-any.whl
+oauth2client==4.1.3
+oauthlib==3.2.2
+odfpy==1.4.1
+olefile==0.47
+omegaconf==2.3.0
+onnx==1.17.0
+openai==1.70.0
+opencv-contrib-python==4.11.0.86
+opencv-python==4.11.0.86
+opencv-python-headless==4.11.0.86
+openpyxl==3.1.5
+openslide-bin==4.0.0.8
+openslide-python==1.4.2
+opentelemetry-api==1.31.1
+opentelemetry-sdk==1.31.1
+opentelemetry-semantic-conventions==0.52b1
+opt_einsum==3.4.0
+optax==0.2.4
+optree==0.14.1
+optuna==4.3.0
+orbax-checkpoint==0.11.10
+orderly-set==5.4.1
+orjson==3.10.16
+osqp==1.0.3
+overrides==7.7.0
+packaging==24.2
+pandas==2.2.3
+pandas-datareader==0.10.0
+pandas-gbq==0.28.0
+pandas-profiling==3.6.6
+pandas-stubs==2.2.2.240909
+pandasql==0.7.3
+pandocfilters==1.5.1
+panel==1.6.2
+papermill==2.6.0
+param==2.2.0
+parso==0.8.4
+parsy==2.1
+partd==1.4.2
+path==17.1.0
+path.py==12.5.0
+pathlib==1.0.1
+pathos==0.3.2
+patsy==1.0.1
+pdf2image==1.17.0
+peewee==3.17.9
+peft==0.14.0
+pettingzoo==1.24.0
+pexpect==4.9.0
+phik==0.12.4
+pickleshare==0.7.5
+pillow==11.1.0
+plac==1.4.5
+platformdirs==4.3.8
+plotly==5.24.1
+plotly-express==0.4.1
+plotnine==0.14.5
+pluggy==1.5.0
+plum-dispatch==2.5.7
+ply==3.11
+polars==1.21.0
+pooch==1.8.2
+portpicker==1.5.2
+pox==0.3.6
+ppft==1.7.7
+preprocessing==0.1.13
+preshed==3.0.9
+prettytable==3.16.0
+proglog==0.1.11
+progressbar2==4.5.0
+prometheus_client==0.21.1
+promise==2.3
+prompt_toolkit==3.0.50
+propcache==0.3.1
+prophet==1.1.6
+proto-plus==1.26.1
+protobuf==5.29.5
+psutil==7.0.0
+psycopg2==2.9.10
+ptyprocess==0.7.0
+pudb==2025.1
+puremagic==1.29
+py-cpuinfo==9.0.0
+py4j==0.10.9.7
+pyaml==25.1.0
+pyannote.core==5.0.0
+pyannote.database==5.1.3
+pyannote.metrics==3.2.1
+PyArabic==0.6.15
+pyarrow==19.0.1
+pyasn1==0.6.1
+pyasn1_modules==0.4.2
+pybind11==2.13.6
+pycairo==1.27.0
+pyclipper==1.3.0.post6
+pycocotools==2.0.8
+pycparser==2.22
+pycryptodome==3.22.0
+pycryptodomex==3.22.0
+pycuda==2025.1
+pydantic==2.11.4
+pydantic_core==2.33.2
+pydata-google-auth==1.9.1
+pydegensac==0.1.2
+pydicom==3.0.1
+pydot==3.0.4
+pydotplus==2.0.2
+PyDrive==1.3.1
+PyDrive2==1.21.3
+pydub==0.25.1
+pyemd==1.0.0
+pyerfa==2.0.1.5
+pyexcel-io==0.6.7
+pyexcel-ods==0.6.0
+pygame==2.6.1
+pygit2==1.17.0
+pygltflib==1.16.4
+Pygments==2.19.1
+PyGObject==3.42.0
+PyJWT==2.10.1
+pyLDAvis==3.4.1
+pylibcudf-cu12==25.2.2
+pylibcugraph-cu12==25.2.0
+pylibraft-cu12==25.2.0
+pyloudnorm==0.1.1
+pymc==5.21.2
+pymc3==3.11.4
+pymongo==4.12.1
+Pympler==1.1
+pymystem3==0.2.0
+pynndescent==0.5.13
+pynvjitlink-cu12==0.5.2
+pynvml==12.0.0
+pyogrio==0.10.0
+Pyomo==6.8.2
+PyOpenGL==3.1.9
+pyOpenSSL==25.0.0
+pyparsing==3.0.9
+pypdf==5.4.0
+pyperclip==1.9.0
+pyproj==3.7.1
+pyshp==2.3.1
+PySocks==1.7.1
+pyspark==3.5.5
+pytensor==2.30.2
+pytesseract==0.3.13
+pytest==8.3.5
+python-apt==0.0.0
+python-bidi==0.6.6
+python-box==7.3.2
+python-dateutil==2.9.0.post0
+python-dotenv==1.1.1
+python-json-logger==3.3.0
+python-louvain==0.16
+python-lsp-jsonrpc==1.1.2
+python-lsp-server==1.12.2
+python-slugify==8.0.4
+python-snappy==0.7.3
+python-utils==3.9.1
+pytools==2025.1.3
+pytorch-ignite==0.5.2
+pytorch-lightning==2.5.1.post0
+pytz==2025.2
+PyUpSet==0.1.1.post7
+pyviz_comms==3.0.4
+PyWavelets==1.8.0
+PyYAML==6.0.2
+pyzmq==24.0.1
+qgrid==1.3.1
+qtconsole==5.6.1
+QtPy==2.4.3
+raft-dask-cu12==25.2.0
+RapidFuzz==3.14.1
+rapids-dask-dependency==25.2.0
+ratelim==0.1.6
+ray==2.46.0
+referencing==0.36.2
+regex==2024.11.6
+requests==2.32.3
+requests-oauthlib==2.0.0
+requests-toolbelt==1.0.0
+requirements-parser==0.9.0
+resampy==0.4.3
+rfc3161-client==1.0.1
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rfc8785==0.1.4
+rgf-python==3.12.0
+rich==14.0.0
+rmm-cu12==25.2.0
+roman-numerals-py==3.1.0
+rpds-py==0.24.0
+rpy2==3.5.17
+rsa==4.9.1
+rtree==1.4.0
+ruamel.yaml==0.18.15
+ruamel.yaml.clib==0.2.14
+s3fs==0.4.2
+s3transfer==0.12.0
+sacremoses==0.1.1
+safetensors==0.5.3
+scikit-image==0.25.2
+scikit-learn==1.2.2
+scikit-learn-intelex==2025.5.0
+scikit-multilearn==0.2.0
+scikit-optimize==0.10.2
+scikit-plot==0.3.7
+scikit-surprise==1.1.4
+scipy==1.15.2
+scooby==0.10.0
+scs==3.2.7.post2
+seaborn==0.12.2
+SecretStorage==3.3.1
+securesystemslib==1.3.0
+segment_anything @ git+https://github.com/facebookresearch/segment-anything.git@dca509fe793f601edb92606367a655c15ac00fdf
+semver==3.0.4
+Send2Trash==1.8.3
+sentence-transformers==3.4.1
+sentencepiece==0.2.0
+sentry-sdk==2.25.1
+setproctitle==1.3.5
+setuptools-scm==8.3.1
+shap==0.44.1
+shapely==2.1.0
+shellingham==1.5.4
+Shimmy==1.3.0
+sigstore==3.6.2
+sigstore-protobuf-specs==0.3.2
+sigstore-rekor-types==0.0.18
+simple-parsing==0.1.7
+simpleitk==2.5.0
+simplejson==3.20.1
+simsimd==6.2.1
+siphash24==1.7
+six==1.17.0
+sklearn-compat==0.1.3
+sklearn-pandas==2.2.0
+slicer==0.0.7
+smart-open==7.1.0
+smmap==5.0.2
+sniffio==1.3.1
+snowballstemmer==2.2.0
+sortedcontainers==2.4.0
+soundfile==0.13.1
+soupsieve==2.6
+sox==1.5.0
+soxr==0.5.0.post1
+spacy==3.8.5
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spanner-graph-notebook==1.1.6
+Sphinx==8.2.3
+sphinx-rtd-theme==0.2.4
+sphinxcontrib-applehelp==2.0.0
+sphinxcontrib-devhelp==2.0.0
+sphinxcontrib-htmlhelp==2.1.0
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==2.0.0
+sphinxcontrib-serializinghtml==2.0.0
+SQLAlchemy==2.0.40
+sqlglot==25.20.2
+sqlparse==0.5.3
+squarify==0.4.4
+srsly==2.5.1
+stable-baselines3==2.1.0
+stanio==0.5.1
+statsmodels==0.14.4
+stopit==1.1.2
+stringzilla==3.12.3
+stumpy==1.13.0
+sympy==1.13.1
+tables==3.10.2
+tabulate==0.9.0
+tbb==2022.1.0
+tbb4py==2022.1.0
+tblib==3.1.0
+tcmlib==1.3.0
+tenacity==9.1.2
+tensorboard==2.18.0
+tensorboard-data-server==0.7.2
+tensorflow==2.18.0
+tensorflow-cloud==0.1.5
+tensorflow-datasets==4.9.8
+tensorflow-hub==0.16.1
+tensorflow-io==0.37.1
+tensorflow-io-gcs-filesystem==0.37.1
+tensorflow-metadata==1.17.0
+tensorflow-probability==0.25.0
+tensorflow-text==2.18.1
+tensorflow_decision_forests==1.11.0
+tensorstore==0.1.73
+termcolor==3.0.1
+terminado==0.18.1
+testpath==0.6.0
+text-unidecode==1.3
+textblob==0.19.0
+texterrors==0.5.1
+texttable==1.7.0
+tf-slim==1.1.0
+tf_keras==2.18.0
+Theano==1.0.5
+Theano-PyMC==1.1.2
+thinc==8.3.4
+threadpoolctl==3.6.0
+tifffile==2025.3.30
+tiktoken==0.9.0
+timm==1.0.15
+tinycss2==1.4.0
+tokenizers==0.21.1
+toml==0.10.2
+toolz==1.0.0
+torch @ https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp311-cp311-linux_x86_64.whl
+torchao==0.10.0
+torchaudio @ https://download.pytorch.org/whl/cu124/torchaudio-2.6.0%2Bcu124-cp311-cp311-linux_x86_64.whl
+torchdata==0.11.0
+torchinfo==1.8.0
+torchmetrics==1.7.1
+torchsummary==1.5.1
+torchtune==0.6.1
+torchvision @ https://download.pytorch.org/whl/cu124/torchvision-0.21.0%2Bcu124-cp311-cp311-linux_x86_64.whl
+tornado==6.4.2
+TPOT==0.12.1
+tqdm==4.67.1
+traitlets==5.7.1
+traittypes==0.2.1
+transformers==4.53.3
+treelite==4.4.1
+treescope==0.1.9
+triton==3.2.0
+trx-python==0.3
+tsfresh==0.21.0
+tuf==6.0.0
+tweepy==4.15.0
+typeguard==4.4.2
+typer==0.15.2
+types-python-dateutil==2.9.0.20241206
+types-pytz==2025.2.0.20250326
+types-setuptools==78.1.0.20250329
+typing-inspect==0.9.0
+typing-inspection==0.4.0
+typing_extensions==4.13.2
+tzdata==2025.2
+tzlocal==5.3.1
+uc-micro-py==1.0.3
+ucx-py-cu12==0.42.0
+ucxx-cu12==0.42.0
+ujson==5.10.0
+umap-learn==0.5.7
+umf==0.10.0
+update-checker==0.18.0
+uri-template==1.3.0
+uritemplate==4.1.1
+urllib3==2.4.0
+urwid==3.0.2
+urwid_readline==0.15.1
+vega-datasets==0.9.0
+visions==0.8.1
+vtk==9.3.1
+wadllib==1.3.6
+Wand==0.6.13
+wandb==0.19.9
+wasabi==1.1.3
+watchdog==6.0.0
+wavio==0.0.9
+wcwidth==0.2.13
+weasel==0.4.1
+webcolors==24.11.1
+webdataset==1.0.2
+webencodings==0.5.1
+websocket-client==1.8.0
+websockets==15.0.1
+Werkzeug==3.1.3
+wget==3.2
+whisper_normalizer==0.1.12
+widgetsnbextension==4.0.14
+woodwork==0.31.0
+wordcloud==1.9.4
+wrapt==1.17.2
+wurlitzer==3.1.1
+xarray==2025.1.2
+xarray-einstats==0.8.0
+xgboost==2.0.3
+xlrd==2.0.1
+xvfbwrapper==0.2.13
+xxhash==3.5.0
+xyzservices==2025.1.0
+y-py==0.6.2
+yarl==1.20.0
+ydata-profiling==4.16.1
+ydf==0.9.0
+yellowbrick==1.5
+yfinance==0.2.55
+ypy-websocket==0.8.4
+zict==3.0.0
+zipp==3.21.0
+zstandard==0.23.0
+

test_conformer_10-15_07-53/model_avg.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74968c063d914760e09e6968d247984902330fba46dbebdfc40d0e1558eea663
+size 52385082

test_conformer_10-15_07-53/nemo_model_avg.nemo

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4639846bbd38855d4001d6eecf550e3d1955038d12914939257b9e1b4dc9c63f
+size 52449280