20250128

config.json

@@ -1,29 +1,46 @@
 {
-  "_name_or_path": "assets/facebookai_xlm_roberta_base",
+  "_name_or_path": "assets/modern_bert_multilingual/20250128/nodecay",
   "architectures": [
-    "XLMRobertaForMaskedLM"
+    "ModernBertForMaskedLM"
   ],
-  "attention_probs_dropout_prob": 0.1,
-  "bos_token_id": 0,
-  "classifier_dropout": null,
-  "eos_token_id": 2,
-  "hidden_act": "gelu",
-  "hidden_dropout_prob": 0.1,
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 151644,
+  "classifier_activation": "gelu",
+  "classifier_bias": false,
+  "classifier_dropout": 0.0,
+  "classifier_pooling": "mean",
+  "cls_token_id": 151644,
+  "decoder_bias": true,
+  "deterministic_flash_attn": false,
+  "embedding_dropout": 0.0,
+  "eos_token_id": 151645,
+  "global_attn_every_n_layers": 3,
+  "global_rope_theta": 160000.0,
+  "gradient_checkpointing": false,
+  "hidden_activation": "gelu",
   "hidden_size": 768,
+  "initializer_cutoff_factor": 2.0,
   "initializer_range": 0.02,
-  "intermediate_size": 3072,
+  "intermediate_size": 1152,
   "layer_norm_eps": 1e-05,
-  "max_position_embeddings": 514,
-  "model_type": "xlm-roberta",
+  "local_attention": 128,
+  "local_rope_theta": 10000.0,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mlp_dropout": 0.0,
+  "model_type": "modernbert",
+  "norm_bias": false,
+  "norm_eps": 1e-05,
   "num_attention_heads": 12,
-  "num_hidden_layers": 12,
-  "output_past": true,
-  "pad_token_id": 1,
+  "num_hidden_layers": 22,
+  "pad_token_id": 151646,
   "position_embedding_type": "absolute",
-  "reference_compile": null,
+  "repad_logits_with_grad": false,
+  "sep_token_id": 151645,
+  "sparse_pred_ignore_index": -100,
+  "sparse_prediction": false,
   "torch_dtype": "float32",
-  "transformers_version": "4.48.0",
-  "type_vocab_size": 1,
-  "use_cache": true,
-  "vocab_size": 250002
+  "transformers_version": "4.48.1",
+  "vocab_size": 151680
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b2e50fb6e5cb06134cfd7be0ba9a6e01675ef132bdca1718c3aecc1c32fb842a
-size 1113205088
+oid sha256:eaef7911078aa9b1d61398ac0e6a1125853bacf37ebe55eb97f6d322f3c2a5d5
+size 910270752

special_tokens_map.json

@@ -1,15 +1,37 @@
 {
-  "bos_token": "<s>",
-  "cls_token": "<s>",
-  "eos_token": "</s>",
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "mask_token": {
-    "content": "<mask>",
-    "lstrip": true,
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "<pad>",
-  "sep_token": "</s>",
-  "unk_token": "<unk>"
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a56def25aa40facc030ea8b0b87f3688e4b3c39eb8b45d5702b3a1300fe2a20
-size 17082734
+oid sha256:6ad6771f60dfa1770ddaad1fe84a7134b6258294bd5357e2998defb03f5233b4
+size 11426146

tokenizer_config.json

@@ -1,56 +1,314 @@
 {
   "added_tokens_decoder": {
-    "0": {
-      "content": "<s>",
+    "151643": {
+      "content": "[UNK]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "1": {
-      "content": "<pad>",
+    "151644": {
+      "content": "[CLS]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "2": {
-      "content": "</s>",
+    "151645": {
+      "content": "[SEP]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "3": {
-      "content": "<unk>",
+    "151646": {
+      "content": "[PAD]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "250001": {
-      "content": "<mask>",
-      "lstrip": true,
+    "151647": {
+      "content": "[MASK]",
+      "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "151648": {
+      "content": "[UNUSED_1]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151649": {
+      "content": "[UNUSED_2]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151650": {
+      "content": "[UNUSED_3]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151651": {
+      "content": "[UNUSED_4]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151652": {
+      "content": "[UNUSED_5]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151653": {
+      "content": "[UNUSED_6]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151654": {
+      "content": "[UNUSED_7]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151655": {
+      "content": "[UNUSED_8]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151656": {
+      "content": "[UNUSED_9]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151657": {
+      "content": "[UNUSED_10]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "[UNUSED_11]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "[UNUSED_12]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "[UNUSED_13]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "[UNUSED_14]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "[UNUSED_15]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "[UNUSED_16]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "[UNUSED_17]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "[UNUSED_18]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "[UNUSED_19]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "[UNUSED_20]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "[UNUSED_21]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151669": {
+      "content": "[UNUSED_22]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151670": {
+      "content": "[UNUSED_23]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151671": {
+      "content": "[UNUSED_24]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151672": {
+      "content": "[UNUSED_25]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151673": {
+      "content": "[UNUSED_26]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151674": {
+      "content": "[UNUSED_27]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151675": {
+      "content": "[UNUSED_28]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151676": {
+      "content": "[UNUSED_29]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151677": {
+      "content": "[UNUSED_30]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151678": {
+      "content": "[UNUSED_31]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151679": {
+      "content": "[UNUSED_32]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
     }
   },
-  "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,
-  "cls_token": "<s>",
+  "cls_token": "[CLS]",
   "do_lower_case": false,
-  "eos_token": "</s>",
   "extra_special_tokens": {},
-  "mask_token": "<mask>",
-  "model_max_length": 512,
-  "pad_token": "<pad>",
-  "sep_token": "</s>",
-  "tokenizer_class": "XLMRobertaTokenizer",
-  "unk_token": "<unk>"
+  "mask_token": "[MASK]",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 8192,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "[UNK]"
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d0248bcc08258853752fa496fa7a411c44ef6785f0a082234eb6edfe80d95f73
+oid sha256:08bef329c1128512ab4e1068ec66ff5a070db0952cdf5879410fab59eb8fbe91
 size 5432

training_args.json

@@ -1,5 +1,5 @@
 {
-    "output_dir": "output/facebookai_xlm_roberta_base_pt_e1",
+    "output_dir": "output/keyword_gacha_multilingual/20250128",
     "overwrite_output_dir": false,
     "do_train": false,
     "do_eval": true,
@@ -10,12 +10,12 @@
     "per_device_eval_batch_size": 8,
     "per_gpu_train_batch_size": null,
     "per_gpu_eval_batch_size": null,
-    "gradient_accumulation_steps": 32,
+    "gradient_accumulation_steps": 224,
     "eval_accumulation_steps": null,
     "eval_delay": 0,
     "torch_empty_cache_steps": null,
-    "learning_rate": 5e-05,
-    "weight_decay": 0.01,
+    "learning_rate": 0.0005,
+    "weight_decay": 1e-05,
     "adam_beta1": 0.9,
     "adam_beta2": 0.999,
     "adam_epsilon": 1e-08,
@@ -24,21 +24,21 @@
     "max_steps": -1,
     "lr_scheduler_type": "warmup_stable_decay",
     "lr_scheduler_kwargs": {
-        "num_decay_steps": 1562,
-        "num_stable_steps": 12494
+        "num_decay_steps": 1114,
+        "num_stable_steps": 0
     },
-    "warmup_ratio": 0.1,
+    "warmup_ratio": 0,
     "warmup_steps": 0,
     "log_level": "passive",
     "log_level_replica": "warning",
     "log_on_each_node": true,
-    "logging_dir": "output/facebookai_xlm_roberta_base_pt_e1/runs/Jan18_11-08-32_Neavo-PC",
+    "logging_dir": "output/keyword_gacha_multilingual/20250128/runs/Jan28_23-39-29_Neavo-PC",
     "logging_strategy": "steps",
     "logging_first_step": false,
-    "logging_steps": 5,
+    "logging_steps": 1,
     "logging_nan_inf_filter": true,
-    "save_strategy": "no",
-    "save_steps": 0,
+    "save_strategy": "steps",
+    "save_steps": 50,
     "save_total_limit": null,
     "save_safetensors": true,
     "save_on_each_node": false,
@@ -64,11 +64,11 @@
     "tpu_metrics_debug": false,
     "debug": [],
     "dataloader_drop_last": false,
-    "eval_steps": 200,
-    "dataloader_num_workers": 4,
-    "dataloader_prefetch_factor": 8,
+    "eval_steps": 50,
+    "dataloader_num_workers": 8,
+    "dataloader_prefetch_factor": null,
     "past_index": -1,
-    "run_name": "output/facebookai_xlm_roberta_base_pt_e1",
+    "run_name": "output/keyword_gacha_multilingual/20250128",
     "disable_tqdm": false,
     "remove_unused_columns": true,
     "label_names": null,
@@ -107,7 +107,7 @@
     "ddp_bucket_cap_mb": null,
     "ddp_broadcast_buffers": null,
     "dataloader_pin_memory": true,
-    "dataloader_persistent_workers": true,
+    "dataloader_persistent_workers": false,
     "skip_memory_metrics": true,
     "use_legacy_prediction_loop": false,
     "push_to_hub": false,